From ca293514082885f61a71d89cdaf2a8331306cc6e Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Thu, 16 Jul 2026 11:42:30 -0700 Subject: [PATCH 01/67] fix: Open up API to swap out pointers --- src/c2pa/c2pa.py | 43 ++++++++++++++++++++++++++--------------- tests/perf/scenarios.py | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 8b252bdc..b6e0523e 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -281,6 +281,28 @@ def _mark_consumed(self): self._handle = None self._lifecycle_state = LifecycleState.CLOSED + def _activate(self, handle, **extra_attrs): + """Attach a native handle (and any extra instance attrs) to self and + mark it ACTIVE. + + Caller must guarantee `handle` is non-null and that ownership is + being transferred here (exactly one activation per handle). + """ + for attr, value in extra_attrs.items(): + setattr(self, attr, value) + self._handle = handle + self._lifecycle_state = LifecycleState.ACTIVE + + @classmethod + def _wrap_native_handle(cls, handle, **extra_attrs): + """Build a brand-new instance around an already-valid, already-owned + native handle, bypassing __init__ entirely. + """ + obj = object.__new__(cls) + ManagedResource.__init__(obj) + obj._activate(handle, **extra_attrs) + return obj + def _cleanup_resources(self): """Release native resources idempotently.""" try: @@ -2876,13 +2898,10 @@ def __init__(self, signer_ptr: ctypes.POINTER(C2paSigner)): """ super().__init__() - self._callback_cb = None - if not signer_ptr: raise C2paError("Invalid signer pointer: pointer is null") - self._handle = signer_ptr - self._lifecycle_state = LifecycleState.ACTIVE + self._activate(signer_ptr, _callback_cb=None) def _release(self): """Release Signer-specific resources (callback reference).""" @@ -3005,25 +3024,17 @@ def from_archive( C2paError: If there was an error creating the builder from archive """ - # Handle builder._handle lifecycle somewhat manually - builder = object.__new__(cls) - ManagedResource.__init__(builder) - builder._context = None - builder._has_context_signer = False - stream_obj = Stream(stream) try: - builder._handle = ( - _lib.c2pa_builder_from_archive(stream_obj._stream) - ) + handle = _lib.c2pa_builder_from_archive(stream_obj._stream) - _check_ffi_operation_result(builder._handle, + _check_ffi_operation_result(handle, "Failed to create builder from archive" ) - builder._lifecycle_state = LifecycleState.ACTIVE - return builder + return cls._wrap_native_handle( + handle, _context=None, _has_context_signer=False) finally: stream_obj.close() diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index d2baae19..46865f4b 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -477,6 +477,28 @@ def scenario_builder_sign_jpeg_archive_roundtrip(iterations: int = 100) -> None: builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) +def scenario_builder_from_archive_roundtrip(iterations: int = 100) -> None: + """Loop Builder.from_archive() itself (context-less alternate constructor), + then sign. Regression guard for the classmethod's native-handle wrapping. + """ + signer = _make_signer() + source_bytes = SOURCE_JPEG.read_bytes() + ingredient_bytes = SIGNED_JPEG.read_bytes() + archive_bytes = io.BytesIO() + Builder(MANIFEST_BASE).to_archive(archive_bytes) + archive_bytes = archive_bytes.getvalue() + for _ in _iterate(iterations): + # from_archive() yields a context-less Builder, so sign() needs an + # explicit signer (no Context to pull one from). + builder = Builder.from_archive(io.BytesIO(archive_bytes)) + with io.BytesIO(ingredient_bytes) as ing: + builder.add_ingredient( + {"relationship": "parentOf", "instance_id": _PARENT_ID}, + "image/jpeg", ing, + ) + builder.sign(signer, "image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) + + # Archive scenarios: builder as working store (to_archive/with_archive) and # per-ingredient archives (write_ingredient_archive/add_ingredient_from_archive). @@ -678,6 +700,18 @@ def scenario_reader_string_apis(iterations: int = 100) -> None: reader.close() +def scenario_signer_construction(iterations: int = 100) -> None: + """Loop Signer.from_info()/__init__ construction and teardown. + + Every other scenario calls _make_signer() once outside its loop, so + repeated Signer construction/destruction has no coverage elsewhere. + Regression guard for Signer.__init__'s native-handle activation. + """ + for _ in _iterate(iterations): + signer = _make_signer() + signer.close() + + # jpeg + png context variants, paired with the `_legacy` scenarios above for # side-by-side comparison. @@ -916,6 +950,7 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "builder_sign_jpeg_two_components_same_mime": scenario_builder_sign_jpeg_two_components_same_mime, "builder_sign_jpeg_two_components_mixed_mime": scenario_builder_sign_jpeg_two_components_mixed_mime, "builder_sign_jpeg_archive_roundtrip": scenario_builder_sign_jpeg_archive_roundtrip, + "builder_from_archive_roundtrip": scenario_builder_from_archive_roundtrip, "builder_to_archive_with_ingredient": scenario_builder_to_archive_with_ingredient, "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": scenario_builder_sign_jpeg_archive_roundtrip_ingredient_in_archive, "builder_write_ingredient_archive": scenario_builder_write_ingredient_archive, @@ -925,6 +960,7 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "reader_error_no_manifest": scenario_reader_error_no_manifest, "builder_error_invalid_manifest": scenario_builder_error_invalid_manifest, "reader_string_apis": scenario_reader_string_apis, + "signer_construction": scenario_signer_construction, "fork_reader_collect": scenario_fork_reader_collect, "fork_contended_mutex": scenario_fork_contended_mutex, "fork_thread_local_orphan": scenario_fork_thread_local_orphan, From af97d7f76fddc9957685a6c9549f9812e62bfdb7 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:45:36 -0700 Subject: [PATCH 02/67] fix: Hardening native handles --- src/c2pa/c2pa.py | 161 +++++++++++++++++------ tests/perf/baseline.json | 20 +++ tests/perf/scenarios.py | 47 +++++++ tests/test_unit_tests.py | 272 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 461 insertions(+), 39 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 0bceabf3..83af9431 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -230,8 +230,13 @@ class ManagedResource: for native resources (e.g. pointers). Subclasses must: - - Set `self._handle` to the native pointer after creation. - - Set `self._lifecycle_state = LifecycleState.ACTIVE` once initialized. + - Call `_activate(handle)` once the native pointer is created and + validated, which takes ownership of it and marks the resource ACTIVE. + Never assign `self._handle` or `self._lifecycle_state` directly. + - Call `_swap_handle(new_handle)` instead when an FFI call consumed the + current handle and returned a replacement. + - Call `_mark_consumed()` when an FFI call took ownership of the handle + without returning a replacement. - Override `_release()` to free class-specific resources (streams, caches, callbacks, etc.), called before the native pointer is freed. @@ -287,18 +292,84 @@ def _activate(self, handle, **extra_attrs): """Attach a native handle (and any extra instance attrs) to self and mark it ACTIVE. - Caller must guarantee `handle` is non-null and that ownership is - being transferred here (exactly one activation per handle). + Ownership of `handle` transfers here: this object frees it on close. + Only an UNINITIALIZED resource can be activated, so a handle can never + be activated twice (which would leak the one being replaced) and a + CLOSED resource can never be resurrected (which would free its handle + a second time). + + Any attribute the subclass's `_release()` reads must be passed in + `extra_attrs` when __init__ did not already set it, otherwise + `_release()` raises during cleanup. + + Args: + handle: Non-null native pointer to take ownership of + **extra_attrs: Instance attributes to set before activating + + Raises: + C2paError: If the handle is null or the resource is not + UNINITIALIZED """ + name = type(self).__name__ + # Guards run before any mutation: a rejected activation must leave + # the object exactly as it was. + if not handle: + raise C2paError(f"{name}: cannot activate a null handle") + if self._lifecycle_state != LifecycleState.UNINITIALIZED: + raise C2paError( + f"{name}: already activated " + f"({self._lifecycle_state.name})") + for attr, value in extra_attrs.items(): setattr(self, attr, value) self._handle = handle self._lifecycle_state = LifecycleState.ACTIVE + def _swap_handle(self, new_handle): + """Replace the handle after an FFI call consumed the old one and + returned a replacement (the consume-and-return pattern, e.g. + c2pa_builder_with_archive / c2pa_reader_with_fragment). + + The old pointer is already owned and freed by the callee, so it is + deliberately NOT freed here; doing so would be a double-free. + + A null return from such a call is ambiguous (the callee may have + failed validation before taking ownership, or failed the operation + after), so callers must not call this with a null replacement. Treat + that case as consumed via `_mark_consumed()` instead: risking a leak + on a path Python cannot reach beats freeing a pointer whose address + may have been recycled. + + Args: + new_handle: Non-null native pointer returned by the FFI call + + Raises: + C2paError: If the resource is not ACTIVE or new_handle is null + """ + name = type(self).__name__ + if self._lifecycle_state != LifecycleState.ACTIVE: + raise C2paError( + f"{name}: cannot swap the handle of a resource that is not " + f"active ({self._lifecycle_state.name})") + if not new_handle: + raise C2paError(f"{name}: cannot swap in a null handle") + + self._handle = new_handle + @classmethod def _wrap_native_handle(cls, handle, **extra_attrs): """Build a brand-new instance around an already-valid, already-owned native handle, bypassing __init__ entirely. + + Because __init__ is bypassed, every attribute the subclass's + `_release()` reads must be passed in `extra_attrs`. + + Args: + handle: Non-null native pointer to take ownership of + **extra_attrs: Instance attributes to set before activating + + Raises: + C2paError: If the handle is null """ obj = object.__new__(cls) ManagedResource.__init__(obj) @@ -315,7 +386,17 @@ def _cleanup_resources(self): and self._lifecycle_state != LifecycleState.CLOSED ): self._lifecycle_state = LifecycleState.CLOSED - self._release() + # A failing _release() must not skip the free below: + # that would strand the native handle on an object already + # marked CLOSED, making it unreachable and unfreeable. + try: + self._release() + except Exception: + logger.error( + "Failed to release %s resources", + type(self).__name__, + exc_info=True, + ) if hasattr(self, '_handle') and self._handle: try: ManagedResource._free_native_ptr(self._handle) @@ -1297,8 +1378,7 @@ def __init__(self): ManagedResource._free_native_ptr(ptr) raise - self._handle = ptr - self._lifecycle_state = LifecycleState.ACTIVE + self._activate(ptr) @classmethod def from_json(cls, json_str: str) -> 'Settings': @@ -1464,7 +1544,7 @@ def __init__( _check_ffi_operation_result( ptr, "Failed to create Context" ) - self._handle = ptr + self._activate(ptr) else: # Use ContextBuilder for settings/signer builder_ptr = _lib.c2pa_context_builder_new() @@ -1484,33 +1564,33 @@ def __init__( if signer is not None: signer._ensure_valid_state() + # c2pa_context_builder_set_signer takes ownership of the + # signer pointer immediately (Box::from_raw), on its error + # path as well as on success. The Signer is therefore + # consumed below on any result; leaving it owning a freed + # pointer would make any later use of it a use-after-free. + self._signer_callback_cb = signer._callback_cb result = ( _lib.c2pa_context_builder_set_signer( builder_ptr, signer._handle, ) ) + signer._mark_consumed() if result != 0: _parse_operation_result_for_error(None) + self._has_signer = True # Build consumes builder_ptr ptr = ( _lib.c2pa_context_builder_build(builder_ptr) ) builder_ptr = None - self._handle = ptr _check_ffi_operation_result( ptr, "Failed to build Context" ) - # Build succeeded, consume the Signer. - # Keep its callback ref alive on this Context, - # then mark it so it won't double-free the - # pointer the Context now owns. - if signer is not None: - self._signer_callback_cb = signer._callback_cb - signer._mark_consumed() - self._has_signer = True + self._activate(ptr) except Exception: # Free builder if build was not reached if builder_ptr is not None: @@ -1520,8 +1600,6 @@ def __init__( pass raise - self._lifecycle_state = LifecycleState.ACTIVE - def _release(self): """Release Context-specific resources.""" self._signer_callback_cb = None @@ -2224,11 +2302,11 @@ def __init__( with Stream(stream) as stream_obj: self._create_reader( format_bytes, stream_obj, manifest_data) - self._lifecycle_state = LifecycleState.ACTIVE def _create_reader(self, format_bytes, stream_obj, manifest_data=None): - """Create a Reader from a Stream. + """Create a native reader from a Stream and activate this Reader + around it. Args: format_bytes: UTF-8 encoded format/MIME type @@ -2236,7 +2314,7 @@ def _create_reader(self, format_bytes, stream_obj, manifest_data: Optional manifest bytes """ if manifest_data is None: - self._handle = _lib.c2pa_reader_from_stream( + ptr = _lib.c2pa_reader_from_stream( format_bytes, stream_obj._stream) else: if not isinstance(manifest_data, bytes): @@ -2244,7 +2322,7 @@ def _create_reader(self, format_bytes, stream_obj, manifest_array = ( ctypes.c_ubyte * len(manifest_data)).from_buffer_copy(manifest_data) - self._handle = ( + ptr = ( _lib.c2pa_reader_from_manifest_data_and_stream( format_bytes, stream_obj._stream, @@ -2254,9 +2332,11 @@ def _create_reader(self, format_bytes, stream_obj, ) _check_ffi_operation_result( - self._handle, + ptr, Reader._ERROR_MESSAGES['reader_error'].format("Unknown error")) + self._activate(ptr) + def _init_from_file(self, path, format_bytes, manifest_data=None): """Open a file and create a reader from it. @@ -2270,7 +2350,6 @@ def _init_from_file(self, path, format_bytes, self._backing_file = open(path, 'rb') self._own_stream = Stream(self._backing_file) self._create_reader(format_bytes, self._own_stream, manifest_data) - self._lifecycle_state = LifecycleState.ACTIVE except C2paError: self._close_streams() raise @@ -2352,18 +2431,17 @@ def _init_from_context(self, context, format_or_path, self._own_stream._stream, ) - # reader_ptr has been consumed by the FFI call. + # reader_ptr has been consumed by the FFI call (freed by it even + # on failure), so there is nothing to free on the error path. reader_ptr = None - self._handle = new_ptr - _check_ffi_operation_result(new_ptr, Reader._ERROR_MESSAGES[ 'reader_error' ].format("Unknown error") ) - self._lifecycle_state = LifecycleState.ACTIVE + self._activate(new_ptr) except Exception: self._close_streams() raise @@ -2449,13 +2527,15 @@ def with_fragment(self, format: str, stream, frag_obj._stream, ) + # c2pa_reader_with_fragment consumed the old handle. A null return + # leaves no replacement to take ownership of. if not new_ptr: self._mark_consumed() _check_ffi_operation_result(new_ptr, Reader._ERROR_MESSAGES[ 'fragment_error' ].format("Unknown error")) - self._handle = new_ptr + self._swap_handle(new_ptr) # Invalidate caches: processing a new BMFF fragment updates the native # reader's state, which can change the manifest data it returns. @@ -3083,13 +3163,13 @@ def __init__( if context is not None: self._init_from_context(context, json_str) else: - self._handle = _lib.c2pa_builder_from_json(json_str) + ptr = _lib.c2pa_builder_from_json(json_str) _check_ffi_operation_result( - self._handle, + ptr, Builder._ERROR_MESSAGES['builder_error'].format("Unknown error")) - self._lifecycle_state = LifecycleState.ACTIVE + self._activate(ptr) def _init_from_context(self, context, json_str): """Initialize Builder from a ContextProvider. @@ -3114,10 +3194,10 @@ def _init_from_context(self, context, json_str): ManagedResource._free_native_ptr(builder_ptr) raise - # Consume-and-return: builder_ptr is consumed, - # new_ptr is the valid pointer going forward + # Consume-and-return: builder_ptr is consumed (freed by the FFI even + # on failure), new_ptr is the valid pointer going forward. Nothing to + # free here on the error path. new_ptr = _lib.c2pa_builder_with_definition(builder_ptr, json_str) - self._handle = new_ptr _check_ffi_operation_result(new_ptr, Builder._ERROR_MESSAGES[ @@ -3125,6 +3205,8 @@ def _init_from_context(self, context, json_str): ].format("Unknown error") ) + self._activate(new_ptr) + def set_no_embed(self): """Set the no-embed flag. @@ -3404,10 +3486,13 @@ def with_archive(self, stream: Any) -> 'Builder': raise C2paError( f"Error loading archive: {e}" ) - # Old handle consumed by FFI - self._handle = new_ptr + # c2pa_builder_with_archive consumed the old handle. A null return + # leaves no replacement to take ownership of. + if not new_ptr: + self._mark_consumed() _check_ffi_operation_result( new_ptr, "Failed to load archive into builder") + self._swap_handle(new_ptr) return self diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index 7af9ffb5..d4827c25 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -221,5 +221,25 @@ "peak_bytes": 3402893, "leaked_bytes": 3230663, "total_allocations": 93141 + }, + "builder_from_archive_roundtrip": { + "peak_bytes": 14258579, + "leaked_bytes": 3436798, + "total_allocations": 1593617 + }, + "signer_construction": { + "peak_bytes": 3307608, + "leaked_bytes": 3243306, + "total_allocations": 117601 + }, + "builder_with_archive_swap": { + "peak_bytes": 3635924, + "leaked_bytes": 3305070, + "total_allocations": 395447 + }, + "reader_with_fragment_swap": { + "peak_bytes": 3733349, + "leaked_bytes": 3306787, + "total_allocations": 1936244 } } \ No newline at end of file diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index 096efa3e..b913aba7 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -36,6 +36,8 @@ CLOUD_JPEG = FIXTURES_DIR / "cloud.jpg" SOURCE_JPEG = FIXTURES_DIR / "A.jpg" SIGNING_PNG = SIGNING_FIXTURES_DIR / "sample1.png" +DASH_INIT_MP4 = FIXTURES_DIR / "dashinit.mp4" +DASH_FRAGMENT = FIXTURES_DIR / "dash1.m4s" _DST_COMPOSITE = "http://cv.iptc.org/newscodes/digitalsourcetype/compositeWithTrainedAlgorithmicMedia" @@ -477,6 +479,49 @@ def scenario_builder_sign_jpeg_archive_roundtrip(iterations: int = 100) -> None: builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) +def scenario_builder_with_archive_swap(iterations: int = 100) -> None: + """Loop Builder.with_archive(), the consume-and-return FFI path. + + c2pa_builder_with_archive consumes the old native handle and returns a + replacement, so the Python side swaps the pointer without freeing the + consumed one. Freeing it would be a double-free, and failing to adopt the + replacement would leak. The other builder scenarios never swap a live + handle, so neither mistake would show up there. + """ + context = Context(signer=_make_signer()) + archive = io.BytesIO() + Builder(MANIFEST_BASE).to_archive(archive) + archive_bytes = archive.getvalue() + for _ in _iterate(iterations): + builder = Builder(MANIFEST_BASE, context=context) + builder.with_archive(io.BytesIO(archive_bytes)) + builder.close() + + +def scenario_reader_with_fragment_swap(iterations: int = 100) -> None: + """Loop Reader.with_fragment(), the other consume-and-return FFI path. + + Same ownership hand-off as with_archive: c2pa_reader_with_fragment eats + the old reader handle and returns a new one. + """ + init_bytes = DASH_INIT_MP4.read_bytes() + fragment_bytes = DASH_FRAGMENT.read_bytes() + for _ in _iterate(iterations): + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + try: + reader.with_fragment( + "video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes), + ) + except C2paError: + # A failed call consumed the old handle just as a successful one + # would, so the scenario measures both outcomes. + pass + finally: + reader.close() + + def scenario_builder_from_archive_roundtrip(iterations: int = 100) -> None: """Loop Builder.from_archive() itself (context-less alternate constructor), then sign. Regression guard for the classmethod's native-handle wrapping. @@ -978,6 +1023,8 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "builder_sign_jpeg_two_components_mixed_mime": scenario_builder_sign_jpeg_two_components_mixed_mime, "builder_sign_jpeg_archive_roundtrip": scenario_builder_sign_jpeg_archive_roundtrip, "builder_from_archive_roundtrip": scenario_builder_from_archive_roundtrip, + "builder_with_archive_swap": scenario_builder_with_archive_swap, + "reader_with_fragment_swap": scenario_reader_with_fragment_swap, "builder_to_archive_with_ingredient": scenario_builder_to_archive_with_ingredient, "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": scenario_builder_sign_jpeg_archive_roundtrip_ingredient_in_archive, "builder_write_ingredient_archive": scenario_builder_write_ingredient_archive, diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index e0b3289c..67bdc2cc 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -30,7 +30,8 @@ from c2pa import Builder, C2paError as Error, Reader, C2paSigningAlg as SigningAlg, C2paSignerInfo, Signer, sdk_version, C2paBuilderIntent, C2paDigitalSourceType from c2pa import Settings, Context, ContextBuilder, ContextProvider -from c2pa.c2pa import Stream, LifecycleState, load_settings, create_signer, create_signer_from_info, ed25519_sign, format_embeddable +from c2pa.c2pa import Stream, LifecycleState, ManagedResource, load_settings, create_signer, create_signer_from_info, ed25519_sign, format_embeddable +import c2pa.c2pa as c2pa_module PROJECT_PATH = os.getcwd() @@ -6924,5 +6925,274 @@ def test_callbacks_return_minus_one_after_stream_collected(self): self.assertEqual(flush_cb(None), -1) +class _FakeHandleResource(ManagedResource): + """ManagedResource with a fake integer handle, for lifecycle tests that + must not touch the native library.""" + + +class _CallbackHoldingResource(ManagedResource): + """Mimics Signer: its _release() reads an attribute that must have been + supplied by __init__ or by _activate's extra_attrs.""" + + def _release(self): + if self._callback_cb: + self._callback_cb = None + + +class TestManagedResourceLifecycle(unittest.TestCase): + """Lifecycle primitives: _activate, _swap_handle, _wrap_native_handle. + + These use fake integer handles and record calls to _free_native_ptr + instead of freeing anything, so a mistake here cannot crash the process. + """ + + def setUp(self): + self.freed = [] + self._real_free = ManagedResource._free_native_ptr + ManagedResource._free_native_ptr = staticmethod(self.freed.append) + + def tearDown(self): + ManagedResource._free_native_ptr = self._real_free + + # _cleanup_resources: a failing _release() must not strand the handle + + def test_release_failure_still_frees_handle(self): + res = _CallbackHoldingResource() + # _callback_cb is never set, so _release() raises AttributeError. + res._activate(0xBBBB) + + res.close() + + self.assertEqual(self.freed, [0xBBBB], + "handle leaked when _release() raised") + self.assertIsNone(res._handle) + + def test_release_failure_is_logged(self): + res = _CallbackHoldingResource() + res._activate(0xBBBB) + + with self.assertLogs('c2pa', level='ERROR') as captured: + res.close() + + self.assertTrue( + any('Failed to release' in line for line in captured.output), + f"_release() failure was not logged: {captured.output}") + + # _activate guards + + def test_activate_rejects_null_handle(self): + res = _FakeHandleResource() + + with self.assertRaises(Error) as ctx: + res._activate(None) + + self.assertIn("null handle", str(ctx.exception)) + self.assertEqual(res._lifecycle_state, LifecycleState.UNINITIALIZED) + + def test_activate_rejects_double_activation(self): + res = _FakeHandleResource() + res._activate(0x1111) + + with self.assertRaises(Error) as ctx: + res._activate(0x2222) + + self.assertIn("already activated", str(ctx.exception)) + # The first handle is still owned, and is freed exactly once. + self.assertEqual(res._handle, 0x1111) + res.close() + self.assertEqual(self.freed, [0x1111]) + + def test_activate_rejects_reactivation_after_close(self): + res = _FakeHandleResource() + res._activate(0x3333) + res.close() + self.freed.clear() + + with self.assertRaises(Error): + res._activate(0x4444) + + # Staying CLOSED is what prevents a second free of the handle. + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(res._handle) + self.assertEqual(self.freed, []) + + def test_activate_does_not_mutate_on_rejection(self): + res = _FakeHandleResource() + res._activate(0x5555) + + with self.assertRaises(Error): + res._activate(0x6666, _extra='should not be set') + + self.assertFalse(hasattr(res, '_extra'), + "rejected activation still set extra_attrs") + + # _swap_handle + + def test_swap_handle_does_not_free_consumed_handle(self): + res = _FakeHandleResource() + res._activate(0xAAA1) + + res._swap_handle(0xAAA2) + + # The FFI already owns and frees the old pointer, so freeing it here + # would be a double-free. + self.assertEqual(self.freed, []) + self.assertEqual(res._handle, 0xAAA2) + + res.close() + self.assertEqual(self.freed, [0xAAA2]) + + def test_swap_handle_requires_active_resource(self): + uninitialized = _FakeHandleResource() + with self.assertRaises(Error) as ctx: + uninitialized._swap_handle(0x1) + self.assertIn("not active", str(ctx.exception)) + + closed = _FakeHandleResource() + closed._activate(0x2) + closed.close() + with self.assertRaises(Error): + closed._swap_handle(0x3) + + def test_swap_handle_rejects_null_replacement(self): + res = _FakeHandleResource() + res._activate(0x7777) + + with self.assertRaises(Error) as ctx: + res._swap_handle(None) + + self.assertIn("null handle", str(ctx.exception)) + self.assertEqual(res._handle, 0x7777) + self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) + + # _wrap_native_handle + + def test_wrap_native_handle_sets_extra_attrs_and_bypasses_init(self): + seen = [] + + class Probe(ManagedResource): + def __init__(self): + raise AssertionError("__init__ must be bypassed") + + def _release(self): + seen.append(self._tag) + + obj = Probe._wrap_native_handle(0xC0DE, _tag='from extra_attrs') + + self.assertEqual(obj._tag, 'from extra_attrs') + self.assertEqual(obj._lifecycle_state, LifecycleState.ACTIVE) + self.assertTrue(obj.is_valid) + # ManagedResource.__init__ still ran, so fork-safety is intact. + self.assertTrue(hasattr(obj, '_owner_pid')) + + obj.close() + self.assertEqual(seen, ['from extra_attrs'], + "_release() could not see the extra attrs") + + def test_wrap_native_handle_rejects_null(self): + with self.assertRaises(Error): + _FakeHandleResource._wrap_native_handle(None) + + def test_close_after_wrap_is_idempotent(self): + obj = _FakeHandleResource._wrap_native_handle(0xD00D) + + obj.close() + obj.close() + + self.assertEqual(self.freed, [0xD00D], "handle freed more than once") + + +class TestNativeHandleOwnership(unittest.TestCase): + """Ownership hand-offs between Python and the native library, driven by + fault injection at the FFI boundary.""" + + def setUp(self): + self.data_dir = os.path.join(os.path.dirname(__file__), "fixtures") + + def _make_signer(self): + with open(os.path.join(self.data_dir, "es256_certs.pem"), "rb") as f: + certs = f.read() + with open(os.path.join(self.data_dir, "es256_private.key"), "rb") as f: + key = f.read() + return Signer.from_info(C2paSignerInfo( + b"es256", certs, key, b"http://timestamp.digicert.com")) + + def test_signer_init_rejects_null_pointer(self): + with self.assertRaises(Error): + Signer(None) + + def test_builder_from_archive_wraps_handle(self): + archive = io.BytesIO() + Builder({"claim_generator": "test", "format": "image/jpeg"}).to_archive( + archive) + + builder = Builder.from_archive(io.BytesIO(archive.getvalue())) + + self.assertTrue(builder.is_valid) + self.assertIsNone(builder._context) + self.assertFalse(builder._has_context_signer) + builder.close() + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) + + def test_context_build_failure_consumes_signer(self): + # c2pa_context_builder_set_signer takes ownership of the signer + # pointer immediately, so a later build failure must still leave the + # Signer consumed. Otherwise it holds a pointer the native side has + # already freed. + signer = self._make_signer() + real_build = c2pa_module._lib.c2pa_context_builder_build + c2pa_module._lib.c2pa_context_builder_build = lambda ptr: None + try: + with self.assertRaises(Error): + Context(signer=signer) + finally: + c2pa_module._lib.c2pa_context_builder_build = real_build + + self.assertIsNone(signer._handle, + "Signer still holds a pointer the native side freed") + self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) + + # Nothing left to free, so close() must be a no-op. + freed = [] + real_free = ManagedResource._free_native_ptr + ManagedResource._free_native_ptr = staticmethod(freed.append) + try: + signer.close() + finally: + ManagedResource._free_native_ptr = real_free + self.assertEqual(freed, []) + + def test_context_with_signer_consumes_it_on_success(self): + signer = self._make_signer() + + context = Context(signer=signer) + + self.assertTrue(context.is_valid) + self.assertIsNone(signer._handle) + self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) + self.assertTrue(context.has_signer) + context.close() + + def test_construction_failure_leaves_nothing_to_free(self): + # A failed FFI construction must not leave a half-constructed object + # holding a handle. Activation happens after the check, so _handle is + # never set. + real_new = c2pa_module._lib.c2pa_context_new + c2pa_module._lib.c2pa_context_new = lambda: None + try: + with self.assertRaises(Error): + Context() + finally: + c2pa_module._lib.c2pa_context_new = real_new + + real_json = c2pa_module._lib.c2pa_builder_from_json + c2pa_module._lib.c2pa_builder_from_json = lambda j: None + try: + with self.assertRaises(Error): + Builder({"claim_generator": "test"}) + finally: + c2pa_module._lib.c2pa_builder_from_json = real_json + + if __name__ == '__main__': unittest.main(warnings='ignore') From 3c045878b12616acc063e1237157bb69265f7da4 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:11:53 -0700 Subject: [PATCH 03/67] fix: COmments --- src/c2pa/c2pa.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 83af9431..f68e3a48 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -289,14 +289,12 @@ def _mark_consumed(self): self._lifecycle_state = LifecycleState.CLOSED def _activate(self, handle, **extra_attrs): - """Attach a native handle (and any extra instance attrs) to self and - mark it ACTIVE. + """Attach a native handle (and any extra instance attrs) to self + and mark it active. Attaching activates it. Ownership of `handle` transfers here: this object frees it on close. - Only an UNINITIALIZED resource can be activated, so a handle can never - be activated twice (which would leak the one being replaced) and a - CLOSED resource can never be resurrected (which would free its handle - a second time). + Only an uninitialized resource can be activated, so a handle can never + be activated twice and a closed resource can never be reopened. Any attribute the subclass's `_release()` reads must be passed in `extra_attrs` when __init__ did not already set it, otherwise @@ -307,12 +305,11 @@ def _activate(self, handle, **extra_attrs): **extra_attrs: Instance attributes to set before activating Raises: - C2paError: If the handle is null or the resource is not - UNINITIALIZED + C2paError: If the handle is null + or the resource is not uninitialized """ name = type(self).__name__ - # Guards run before any mutation: a rejected activation must leave - # the object exactly as it was. + # A rejected activation must leave the object as it was. if not handle: raise C2paError(f"{name}: cannot activate a null handle") if self._lifecycle_state != LifecycleState.UNINITIALIZED: @@ -327,18 +324,14 @@ def _activate(self, handle, **extra_attrs): def _swap_handle(self, new_handle): """Replace the handle after an FFI call consumed the old one and - returned a replacement (the consume-and-return pattern, e.g. - c2pa_builder_with_archive / c2pa_reader_with_fragment). + returned a replacement. - The old pointer is already owned and freed by the callee, so it is - deliberately NOT freed here; doing so would be a double-free. + The old pointer is already owned and freed by the callee, + so it is not freed here. A null return from such a call is ambiguous (the callee may have failed validation before taking ownership, or failed the operation - after), so callers must not call this with a null replacement. Treat - that case as consumed via `_mark_consumed()` instead: risking a leak - on a path Python cannot reach beats freeing a pointer whose address - may have been recycled. + after), so callers must not call this with a null replacement. Args: new_handle: Non-null native pointer returned by the FFI call @@ -358,8 +351,8 @@ def _swap_handle(self, new_handle): @classmethod def _wrap_native_handle(cls, handle, **extra_attrs): - """Build a brand-new instance around an already-valid, already-owned - native handle, bypassing __init__ entirely. + """Build a brand-new instance around an already-valid, + already-owned native handle, bypassing __init__ entirely. Because __init__ is bypassed, every attribute the subclass's `_release()` reads must be passed in `extra_attrs`. From 9d7b2aca05781aed977dc263266d2c01f11a535f Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:32:34 -0700 Subject: [PATCH 04/67] fix: Update PID stamping --- src/c2pa/c2pa.py | 31 ++- tests/perf/baseline.json | 25 ++ tests/perf/scenarios.py | 156 ++++++++++++ tests/test_unit_tests.py | 403 +++++++++++++++++++++++++++++- tests/test_unit_tests_threaded.py | 156 ++++++++++++ 5 files changed, 765 insertions(+), 6 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index f68e3a48..9ef2f4d3 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -224,6 +224,12 @@ class LifecycleState(enum.IntEnum): CLOSED = 2 +# Attributes that make up the lifecycle state itself, which _activate sets +# from its own arguments and must never accept from a caller's extra_attrs. +_RESERVED_ACTIVATION_ATTRS = frozenset( + {'_handle', '_lifecycle_state', '_owner_pid'}) + + class ManagedResource: """Base class for objects that hold a native (C FFI) resource. This is an internal base class that provides lifecycle management @@ -300,13 +306,19 @@ def _activate(self, handle, **extra_attrs): `extra_attrs` when __init__ did not already set it, otherwise `_release()` raises during cleanup. + `extra_attrs` cannot carry the lifecycle attributes themselves. + `_owner_pid` in particular records the process that created the + handle, and overwriting it would let a forked child free a pointer + its parent still owns. + Args: handle: Non-null native pointer to take ownership of **extra_attrs: Instance attributes to set before activating Raises: - C2paError: If the handle is null - or the resource is not uninitialized + C2paError: If the handle is null, + the resource is not uninitialized, + or extra_attrs names a lifecycle attribute """ name = type(self).__name__ # A rejected activation must leave the object as it was. @@ -316,6 +328,11 @@ def _activate(self, handle, **extra_attrs): raise C2paError( f"{name}: already activated " f"({self._lifecycle_state.name})") + reserved = _RESERVED_ACTIVATION_ATTRS.intersection(extra_attrs) + if reserved: + raise C2paError( + f"{name}: cannot set lifecycle attributes via extra_attrs: " + f"{', '.join(sorted(reserved))}") for attr, value in extra_attrs.items(): setattr(self, attr, value) @@ -356,15 +373,19 @@ def _wrap_native_handle(cls, handle, **extra_attrs): Because __init__ is bypassed, every attribute the subclass's `_release()` reads must be passed in `extra_attrs`. + The lifecycle attributes are still off limits there, + and the instance is stamped with the creating process either way. Args: handle: Non-null native pointer to take ownership of **extra_attrs: Instance attributes to set before activating Raises: - C2paError: If the handle is null + C2paError: If the handle is null, or extra_attrs names a + lifecycle attribute """ obj = object.__new__(cls) + # Stamps _owner_pid, which the fork guard relies on. ManagedResource.__init__(obj) obj._activate(handle, **extra_attrs) return obj @@ -2298,8 +2319,8 @@ def __init__( def _create_reader(self, format_bytes, stream_obj, manifest_data=None): - """Create a native reader from a Stream and activate this Reader - around it. + """Create a native reader from a Stream + and activate this Reader around it. Args: format_bytes: UTF-8 encoded format/MIME type diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index d4827c25..934a6390 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -241,5 +241,30 @@ "peak_bytes": 3733349, "leaked_bytes": 3306787, "total_allocations": 1936244 + }, + "fork_swap_cleanup": { + "peak_bytes": 3639043, + "leaked_bytes": 3308397, + "total_allocations": 401032 + }, + "swap_chain_churn": { + "peak_bytes": 3650097, + "leaked_bytes": 3319300, + "total_allocations": 379064 + }, + "fork_consumed_signer": { + "peak_bytes": 3321464, + "leaked_bytes": 3258207, + "total_allocations": 130030 + }, + "fork_contended_mutex_swap": { + "peak_bytes": 7281970, + "leaked_bytes": 3443799, + "total_allocations": 18799685 + }, + "fork_contended_mutex_wrap": { + "peak_bytes": 7268578, + "leaked_bytes": 3442991, + "total_allocations": 17791158 } } \ No newline at end of file diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index b913aba7..b4b7e46c 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -979,6 +979,157 @@ def _child(): context.close() +def _fork_contended_over(make_object, iterations): + """Fork over an object built by make_object() while 8 threads churn + Readers, so the registry Mutex is likely held at the instant of fork(). + + The child closes the inherited object. Without the PID guard that close + calls into the native library and can block forever on a mutex left + locked by a thread that fork() did not clone, which _fork_wait's alarm + reports as a timeout. The parent closes afterwards for the real free. + """ + if not hasattr(os, "fork"): + return + stop = threading.Event() + + def _worker(): + while not stop.is_set(): + with open(SIGNED_JPEG, "rb") as f: + r = Reader("image/jpeg", f) + r.close() + + threads = [threading.Thread(target=_worker, daemon=True) + for _ in range(8)] + for t in threads: + t.start() + try: + for _ in _iterate(iterations): + for _ in range(5): + obj = make_object() + + def _child(o=obj): + o.close() + gc.collect() + + _fork_wait(_child) + obj.close() + finally: + stop.set() + for t in threads: + t.join(timeout=5) + + +def scenario_fork_contended_mutex_swap(iterations: int = 100) -> None: + """Fork safety benchmark scenario: + fork over a Builder whose handle came from with_archive(), under the same + thread contention as fork_contended_mutex. That scenario only ever forks + over handles that came straight from a constructor, so a swapped-in + handle losing its stamp would go unnoticed there. + """ + if not hasattr(os, "fork"): + return + context = Context(signer=_make_signer()) + archive = io.BytesIO() + Builder(MANIFEST_BASE).to_archive(archive) + archive_bytes = archive.getvalue() + + def _make(): + builder = Builder(MANIFEST_BASE, context=context) + builder.with_archive(io.BytesIO(archive_bytes)) + return builder + + _fork_contended_over(_make, iterations) + + +def scenario_fork_contended_mutex_wrap(iterations: int = 100) -> None: + """Fork safety benchmark scenario: + fork over a Builder built by from_archive(), under thread contention. + from_archive is the only path that bypasses __init__, so it is the one + most likely to be missing the PID stamp the child's close() depends on. + """ + if not hasattr(os, "fork"): + return + archive = io.BytesIO() + Builder(MANIFEST_BASE).to_archive(archive) + archive_bytes = archive.getvalue() + + _fork_contended_over( + lambda: Builder.from_archive(io.BytesIO(archive_bytes)), iterations) + + +def scenario_fork_consumed_signer(iterations: int = 100) -> None: + """Fork safety benchmark scenario: + the parent builds a Context that consumed a Signer, then forks. The child + closes both. The consumed Signer holds no handle, so it must be inert in + either process, and the Context must be skipped by the PID guard. + """ + if not hasattr(os, "fork"): + return + for _ in _iterate(iterations): + signer = _make_signer() + context = Context(signer=signer) + + def _child(c=context, s=signer): + s.close() + c.close() + gc.collect() + + _fork_wait(_child) + signer.close() + context.close() + + +def scenario_swap_chain_churn(iterations: int = 100) -> None: + """Loop with_archive() repeatedly on one Builder, so a chain of handles + is consumed and replaced on a single live object. Every other scenario + swaps a given object at most once. + + This one is a crash and allocation-churn guard rather than a leak gate. + Only one Builder is closed however many times the loop runs, so a + close-path leak here is O(1) and invisible against the interpreter's + allocation floor. What a broken swap does instead is fail loudly: keeping + the consumed pointer makes the next call raise UntrackedPointer from the + native registry, and freeing it makes the free itself fail. total_allocations + still tracks the churn. + """ + context = Context(signer=_make_signer()) + archive = io.BytesIO() + Builder(MANIFEST_BASE).to_archive(archive) + archive_bytes = archive.getvalue() + builder = Builder(MANIFEST_BASE, context=context) + for _ in _iterate(iterations): + builder.with_archive(io.BytesIO(archive_bytes)) + builder.close() + context.close() + + +def scenario_fork_swap_cleanup(iterations: int = 100) -> None: + """Fork safety benchmark scenario: + the handle a Builder owns at fork time came from with_archive(), which + consumed the original and returned a replacement. The child must skip the + free on the swapped-in handle just as it would on an original one, and the + parent must still free it exactly once afterwards. The other fork + scenarios only ever fork over handles that came straight from a + constructor. + """ + if not hasattr(os, "fork"): + return + context = Context(signer=_make_signer()) + archive = io.BytesIO() + Builder(MANIFEST_BASE).to_archive(archive) + archive_bytes = archive.getvalue() + for _ in _iterate(iterations): + builder = Builder(MANIFEST_BASE, context=context) + builder.with_archive(io.BytesIO(archive_bytes)) + + def _child(b=builder): + b.close() + gc.collect() + + _fork_wait(_child) + builder.close() + + def scenario_fork_stream_cleanup(iterations: int = 100) -> None: """Fork safety benchmark scenario: Stream wraps a BytesIO with ctypes callbacks stored as instance attributes. @@ -1042,6 +1193,11 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "fork_parent_frees_after_fork": scenario_fork_parent_frees_after_fork, "fork_child_sys_exit": scenario_fork_child_sys_exit, "fork_stream_cleanup": scenario_fork_stream_cleanup, + "fork_swap_cleanup": scenario_fork_swap_cleanup, + "fork_contended_mutex_swap": scenario_fork_contended_mutex_swap, + "fork_contended_mutex_wrap": scenario_fork_contended_mutex_wrap, + "fork_consumed_signer": scenario_fork_consumed_signer, + "swap_chain_churn": scenario_swap_chain_churn, } diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 67bdc2cc..78154cca 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -6939,8 +6939,20 @@ def _release(self): self._callback_cb = None +class _ReleaseRecordingResource(ManagedResource): + """Records _release() calls for test asserts.""" + + def __init__(self): + super().__init__() + self.release_calls = 0 + + def _release(self): + self.release_calls += 1 + + class TestManagedResourceLifecycle(unittest.TestCase): - """Lifecycle primitives: _activate, _swap_handle, _wrap_native_handle. + """Lifecycle primitives (_activate, _swap_handle, _wrap_native_handle) + and the _owner_pid stamp that governs which process may free a handle. These use fake integer handles and record calls to _free_native_ptr instead of freeing anything, so a mistake here cannot crash the process. @@ -6954,6 +6966,12 @@ def setUp(self): def tearDown(self): ManagedResource._free_native_ptr = self._real_free + def _free_counts(self): + counts = {} + for handle in self.freed: + counts[handle] = counts.get(handle, 0) + 1 + return counts + # _cleanup_resources: a failing _release() must not strand the handle def test_release_failure_still_frees_handle(self): @@ -7101,6 +7119,389 @@ def test_close_after_wrap_is_idempotent(self): self.assertEqual(self.freed, [0xD00D], "handle freed more than once") + def test_every_construction_path_records_owner_pid(self): + pid = os.getpid() + + plain = _FakeHandleResource() + self.assertEqual(plain._owner_pid, pid) + + activated = _FakeHandleResource() + activated._activate(0xA1) + self.assertEqual(activated._owner_pid, pid) + + wrapped = _FakeHandleResource._wrap_native_handle(0xA2) + self.assertEqual(wrapped._owner_pid, pid) + + # A swap keeps the original stamp: + # the replacement handle was allocated by the same process + # that created the object. + wrapped._swap_handle(0xA3) + self.assertEqual(wrapped._owner_pid, pid) + + def test_activate_rejects_reserved_extra_attrs(self): + for attr, value in ( + ('_owner_pid', os.getpid() + 1), + ('_handle', 0xDEAD), + ('_lifecycle_state', LifecycleState.CLOSED), + ): + with self.subTest(attr=attr): + res = _FakeHandleResource() + stamp = res._owner_pid + + with self.assertRaises(Error) as ctx: + res._activate(0xB1, **{attr: value}) + + self.assertIn(attr, str(ctx.exception)) + self.assertEqual(res._owner_pid, stamp) + self.assertEqual( + res._lifecycle_state, LifecycleState.UNINITIALIZED) + self.assertIsNone(res._handle) + + def test_wrap_native_handle_rejects_reserved_extra_attrs(self): + with self.assertRaises(Error): + _FakeHandleResource._wrap_native_handle( + 0xB2, _owner_pid=os.getpid() + 1) + + def test_foreign_child_skips_free_for_wrapped_and_swapped(self): + wrapped = _FakeHandleResource._wrap_native_handle(0xC1) + wrapped._owner_pid = os.getpid() + 1 + wrapped.close() + + swapped = _FakeHandleResource() + swapped._activate(0xC2) + swapped._swap_handle(0xC3) + swapped._owner_pid = os.getpid() + 1 + swapped.close() + + self.assertEqual(self.freed, [], + "forked child freed a pointer its parent still owns") + # The parent still owns these handles, + # so the child must not mark the objects dead either. + self.assertEqual(wrapped._lifecycle_state, LifecycleState.ACTIVE) + self.assertEqual(swapped._handle, 0xC3) + + def test_owning_process_frees_wrapped_and_swapped_exactly_once(self): + wrapped = _FakeHandleResource._wrap_native_handle(0xC4) + wrapped.close() + wrapped.close() + + swapped = _FakeHandleResource() + swapped._activate(0xC5) + swapped._swap_handle(0xC6) + swapped.close() + + # 0xC5 was consumed by the (simulated) FFI swap, + # so only the replacement is ours to free. + self.assertEqual(self._free_counts(), {0xC4: 1, 0xC6: 1}) + + def test_foreign_child_skips_release(self): + foreign = _ReleaseRecordingResource() + foreign._activate(0xD1) + foreign._owner_pid = os.getpid() + 1 + foreign.close() + self.assertEqual(foreign.release_calls, 0) + + owned = _ReleaseRecordingResource() + owned._activate(0xD2) + owned.close() + self.assertEqual(owned.release_calls, 1) + + def test_consumed_resource_frees_nothing_in_either_process(self): + owned = _FakeHandleResource() + owned._activate(0xE1) + owned._mark_consumed() + owned.close() + + foreign = _FakeHandleResource() + foreign._activate(0xE2) + foreign._mark_consumed() + foreign._owner_pid = os.getpid() + 1 + foreign.close() + + self.assertEqual(self.freed, []) + + +class TestManagedResourceObjects(TestContextAPIs): + """Tests native resource handling management when managed manually. + """ + + def _instrument_frees(self): + """Record frees instead of performing them, and restore on teardown.""" + freed = [] + real_free = ManagedResource._free_native_ptr + ManagedResource._free_native_ptr = staticmethod(freed.append) + self.addCleanup( + lambda: setattr( + ManagedResource, '_free_native_ptr', real_free)) + return freed + + def _make_archive(self, manifest=None): + archive = io.BytesIO() + builder = Builder(manifest or self.test_manifest) + try: + builder.to_archive(archive) + finally: + builder.close() + archive.seek(0) + return archive + + # Activation: every public constructor stamps the creating process + + def test_settings_activation_paths(self): + pid = os.getpid() + for label, factory in ( + ("Settings()", lambda: Settings()), + ("from_json", lambda: Settings.from_json('{"version_major": 1}')), + ("from_dict", lambda: Settings.from_dict({"version_major": 1})), + ): + with self.subTest(path=label): + settings = factory() + try: + self.assertTrue(settings.is_valid) + self.assertEqual(settings._owner_pid, pid) + finally: + settings.close() + + def test_context_activation_paths(self): + pid = os.getpid() + settings = Settings.from_dict({"version_major": 1}) + try: + for label, factory in ( + # No settings and no signer takes the c2pa_context_new path. + ("Context()", lambda: Context()), + # Anything else goes through the ContextBuilder path. + ("Context(settings)", lambda: Context(settings)), + ("from_dict", lambda: Context.from_dict({"version_major": 1})), + ("builder()", + lambda: Context.builder().with_settings(settings).build()), + ): + with self.subTest(path=label): + context = factory() + try: + self.assertTrue(context.is_valid) + self.assertEqual(context._owner_pid, pid) + finally: + context.close() + finally: + settings.close() + + def test_reader_activation_paths(self): + pid = os.getpid() + context = Context() + try: + with open(DEFAULT_TEST_FILE, "rb") as f: + from_stream = Reader("image/jpeg", f) + self.addCleanup(from_stream.close) + self.assertEqual(from_stream._owner_pid, pid) + + from_path = Reader(DEFAULT_TEST_FILE) + self.addCleanup(from_path.close) + self.assertEqual(from_path._owner_pid, pid) + + with open(DEFAULT_TEST_FILE, "rb") as f: + with_context = Reader(DEFAULT_TEST_FILE, context=context) + self.addCleanup(with_context.close) + self.assertEqual(with_context._owner_pid, pid) + + with open(DEFAULT_TEST_FILE, "rb") as f: + created = Reader.try_create("image/jpeg", f) + self.addCleanup(created.close) + self.assertEqual(created._owner_pid, pid) + finally: + context.close() + + def test_builder_activation_paths(self): + pid = os.getpid() + context = Context() + try: + plain = Builder(self.test_manifest) + self.addCleanup(plain.close) + self.assertEqual(plain._owner_pid, pid) + + from_json = Builder.from_json(self.test_manifest) + self.addCleanup(from_json.close) + self.assertEqual(from_json._owner_pid, pid) + + with_context = Builder(self.test_manifest, context=context) + self.addCleanup(with_context.close) + self.assertEqual(with_context._owner_pid, pid) + + # from_archive is the only caller of _wrap_native_handle, + # which bypasses __init__ entirely + wrapped = Builder.from_archive(self._make_archive()) + self.addCleanup(wrapped.close) + self.assertEqual(wrapped._owner_pid, pid) + self.assertIsNone(wrapped._context) + self.assertFalse(wrapped._has_context_signer) + finally: + context.close() + + def test_signer_activation_paths(self): + signer = self._ctx_make_signer() + self.addCleanup(signer.close) + self.assertTrue(signer.is_valid) + self.assertEqual(signer._owner_pid, os.getpid()) + + callback_signer = self._ctx_make_callback_signer() + self.addCleanup(callback_signer.close) + self.assertEqual(callback_signer._owner_pid, os.getpid()) + + # Swapping: the two public consume-and-return APIs + + def test_builder_with_archive_swaps_the_handle(self): + context = Context() + self.addCleanup(context.close) + builder = Builder(self.test_manifest, context=context) + original_handle = builder._handle + original_stamp = builder._owner_pid + + result = builder.with_archive(self._make_archive()) + + self.assertIs(result, builder, "with_archive should return self") + self.assertNotEqual(builder._handle, original_handle, + "the native handle was not replaced") + self.assertEqual(builder._lifecycle_state, LifecycleState.ACTIVE) + # The replacement came from this process, so the stamp still applies. + self.assertEqual(builder._owner_pid, original_stamp) + self.assertEqual(builder._owner_pid, os.getpid()) + builder.close() + + def test_reader_with_fragment_swaps_the_handle(self): + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + context = Context() + self.addCleanup(context.close) + + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init, context=context) + self.addCleanup(reader.close) + original_handle = reader._handle + + # The Reader consumed the first handle, so the init stream is reopened. + with open(init_path, "rb") as init, open(fragment_path, "rb") as frag: + result = reader.with_fragment("video/mp4", init, frag) + + self.assertIs(result, reader, "with_fragment should return self") + self.assertNotEqual(reader._handle, original_handle, + "the native handle was not replaced") + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + self.assertEqual(reader._owner_pid, os.getpid()) + + def test_swapped_builder_is_freed_exactly_once(self): + context = Context() + self.addCleanup(context.close) + builder = Builder(self.test_manifest, context=context) + original_handle = builder._handle + # The helper closes a temporary Builder, + # whose free would otherwise be counted here. + archive = self._make_archive() + + # Instrument across the swap so a free of the consumed pointer is recorded. + freed = self._instrument_frees() + builder.with_archive(archive) + swapped_handle = builder._handle + + self.assertEqual(freed, [], "the swap freed the consumed pointer") + + builder.close() + builder.close() + + # Only the replacement is ours to free: the original was consumed by + # the FFI call that returned it. + self.assertEqual(freed, [swapped_handle]) + self.assertNotIn(original_handle, freed) + + def test_repeated_swaps_on_one_builder(self): + # Each with_archive consumes the handle the previous one returned, so + # a chain of swaps is where a wrong swap surfaces: keeping the + # consumed pointer makes the next call raise UntrackedPointer from + # the native registry. + context = Context() + self.addCleanup(context.close) + builder = Builder(self.test_manifest, context=context) + self.addCleanup(builder.close) + + seen = [builder._handle] + for _ in range(5): + builder.with_archive(self._make_archive()) + self.assertEqual(builder._lifecycle_state, LifecycleState.ACTIVE) + seen.append(builder._handle) + + self.assertTrue(builder.is_valid) + self.assertEqual(builder._owner_pid, os.getpid()) + + def test_context_consumes_signer_but_not_settings(self): + settings = Settings.from_dict({"version_major": 1}) + signer = self._ctx_make_signer() + + context = Context(settings=settings, signer=signer) + self.addCleanup(context.close) + + # The signer pointer moved to the native context builder. + self.assertIsNone(signer._handle) + self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) + self.assertTrue(context.has_signer) + + # Settings are copied, not consumed, so the caller still owns them. + self.assertTrue(settings.is_valid) + self.assertEqual(settings._owner_pid, os.getpid()) + settings.close() + + def test_consumed_signer_close_frees_nothing(self): + signer = self._ctx_make_signer() + context = Context(signer=signer) + self.addCleanup(context.close) + + freed = self._instrument_frees() + signer.close() + + self.assertEqual(freed, [], + "closing a consumed Signer freed a pointer the " + "context now owns") + + def test_builder_with_archive_null_return_consumes_self(self): + builder = Builder(self.test_manifest) + real_call = c2pa_module._lib.c2pa_builder_with_archive + c2pa_module._lib.c2pa_builder_with_archive = lambda b, s: None + try: + with self.assertRaises(Error): + builder.with_archive(self._make_archive()) + finally: + c2pa_module._lib.c2pa_builder_with_archive = real_call + + # The FFI consumed the old handle and returned no replacement, + # so there is nothing left for this object to own... + self.assertIsNone(builder._handle) + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) + + freed = self._instrument_frees() + builder.close() + self.assertEqual(freed, []) + + def test_reader_with_fragment_null_return_consumes_self(self): + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = ( + lambda r, f, s, frag: None) + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("video/mp4", init, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + + self.assertIsNone(reader._handle) + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + + freed = self._instrument_frees() + reader.close() + self.assertEqual(freed, []) + class TestNativeHandleOwnership(unittest.TestCase): """Ownership hand-offs between Python and the native library, driven by diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index efb7ba27..44609adc 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -2911,5 +2911,161 @@ def thread_work(thread_id): self.assertNotEqual(current_manifest["active_manifest"], thread_manifest_data[other_thread_id]["active_manifest"]) +class TestManagedResourceCrossThread(unittest.TestCase): + """Tests cross-thread resources handling, especially closing/releasind. + """ + + def setUp(self): + self.freed = [] + self._real_free = ManagedResource._free_native_ptr + ManagedResource._free_native_ptr = staticmethod(self.freed.append) + + def tearDown(self): + ManagedResource._free_native_ptr = self._real_free + + def _free_counts(self): + counts = {} + for handle in self.freed: + counts[handle] = counts.get(handle, 0) + 1 + return counts + + def test_cross_thread_create_and_close_frees_exactly_once(self): + count = 300 + pid = os.getpid() + + def create(index): + res = _ConcreteResource() + res._activate(0x10000 + index) + return res + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + created = list(pool.map(create, range(count))) + + # Created on worker threads, closed on the main thread. + for res in created: + self.assertEqual(res._owner_pid, pid) + self.assertFalse(is_foreign_process(res)) + res.close() + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + made_on_main = [] + for index in range(count): + res = _ConcreteResource() + res._activate(0x20000 + index) + made_on_main.append(res) + # Created on the main thread, closed on worker threads. + list(pool.map(lambda r: r.close(), made_on_main)) + + expected = {0x10000 + i: 1 for i in range(count)} + expected.update({0x20000 + i: 1 for i in range(count)}) + # Restrict to this test's handles: resources dropped by other tests in + # the class can be collected at any point and land in self.freed. + counts = {handle: value + for handle, value in self._free_counts().items() + if handle in expected} + self.assertEqual(counts, expected) + + def test_third_thread_gc_of_dropped_reference_frees_exactly_once(self): + import gc + + def make_and_drop(index): + res = _ConcreteResource() + res._activate(0x30000 + index) + # Reference dies here; __del__ may run on this thread or later. + return index + + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(make_and_drop, range(200))) + + gc.collect() + + # Count only this test's handles: resources dropped by other tests in + # the class can be collected at any point and land in self.freed. + counts = {handle: count + for handle, count in self._free_counts().items() + if 0x30000 <= handle < 0x30000 + 200} + self.assertEqual(len(counts), 200, + "dropped resources were not all freed") + self.assertEqual(set(counts.values()), {1}, + "a dropped resource was freed more than once") + + +class TestSettingsAsContextAcrossThreads(unittest.TestCase): + """Tests Settings handed between threads and reused as the basis + for Contexts, using real native handles. + """ + + def setUp(self): + self.manifest = { + "claim_generator": "threaded_stamp_test", + "format": "image/jpeg", + "assertions": [], + } + + def test_settings_relayed_across_threads_stays_usable(self): + settings = Settings() + pid = os.getpid() + results = [] + errors = [] + + def build_context_and_builder(): + try: + context = Context(settings=settings) + builder = Builder(self.manifest, context=context) + results.append(( + builder._owner_pid, context._owner_pid, builder.is_valid)) + builder.close() + context.close() + except Exception as exc: + errors.append(exc) + + # Sequential hand-off: each thread owns the Settings for its turn. + for _ in range(8): + thread = threading.Thread(target=build_context_and_builder) + thread.start() + thread.join() + + settings.close() + + self.assertEqual(errors, []) + self.assertEqual(len(results), 8) + for builder_pid, context_pid, valid in results: + self.assertEqual(builder_pid, pid) + self.assertEqual(context_pid, pid) + self.assertTrue(valid) + self.assertEqual(settings._owner_pid, pid) + + def test_context_created_on_one_thread_closed_on_another(self): + created = [] + errors = [] + + def create(): + try: + created.append(Context()) + except Exception as exc: + errors.append(exc) + + def close_all(): + try: + for context in created: + context.close() + except Exception as exc: + errors.append(exc) + + maker = threading.Thread(target=create) + maker.start() + maker.join() + + closer = threading.Thread(target=close_all) + closer.start() + closer.join() + + self.assertEqual(errors, []) + self.assertEqual(len(created), 1) + for context in created: + self.assertEqual(context._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(context._handle) + + if __name__ == '__main__': unittest.main() From e926b5f05d8d661f4f4b59e45e33264db908fd42 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:03:46 -0700 Subject: [PATCH 05/67] fix: Make isntances swappable --- tests/test_unit_tests.py | 284 +++++++++++++++++++-------------------- 1 file changed, 138 insertions(+), 146 deletions(-) diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 78154cca..fd7b3669 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -6925,40 +6925,39 @@ def test_callbacks_return_minus_one_after_stream_collected(self): self.assertEqual(flush_cb(None), -1) -class _FakeHandleResource(ManagedResource): - """ManagedResource with a fake integer handle, for lifecycle tests that - must not touch the native library.""" - - -class _CallbackHoldingResource(ManagedResource): - """Mimics Signer: its _release() reads an attribute that must have been - supplied by __init__ or by _activate's extra_attrs.""" - - def _release(self): - if self._callback_cb: - self._callback_cb = None +class TestManagedResourceLifecycle(unittest.TestCase): + """Lifecycle primitives (_activate, _swap_handle, _wrap_native_handle), + the _owner_pid stamp that governs which process may free a handle, and + the ownership hand-offs between Python and the native library. + setUp records frees instead of performing them, so a miscount reads as a + leak or a double-free rather than a crash. Tests holding real handles + call _use_real_frees() first. + """ -class _ReleaseRecordingResource(ManagedResource): - """Records _release() calls for test asserts.""" + class _FakeHandleResource(ManagedResource): + """Concrete subclass with no resources of its own.""" - def __init__(self): - super().__init__() - self.release_calls = 0 + class _CallbackHoldingResource(ManagedResource): + """Mimics Signer: its _release() reads an attribute that must have + been supplied by __init__ or by _activate's extra_attrs.""" - def _release(self): - self.release_calls += 1 + def _release(self): + if self._callback_cb: + self._callback_cb = None + class _ReleaseRecordingResource(ManagedResource): + """Records _release() calls for test asserts.""" -class TestManagedResourceLifecycle(unittest.TestCase): - """Lifecycle primitives (_activate, _swap_handle, _wrap_native_handle) - and the _owner_pid stamp that governs which process may free a handle. + def __init__(self): + super().__init__() + self.release_calls = 0 - These use fake integer handles and record calls to _free_native_ptr - instead of freeing anything, so a mistake here cannot crash the process. - """ + def _release(self): + self.release_calls += 1 def setUp(self): + self.data_dir = FIXTURES_DIR self.freed = [] self._real_free = ManagedResource._free_native_ptr ManagedResource._free_native_ptr = staticmethod(self.freed.append) @@ -6972,10 +6971,22 @@ def _free_counts(self): counts[handle] = counts.get(handle, 0) + 1 return counts + def _use_real_frees(self): + """Undo setUp's recorder, so native handles are really freed.""" + ManagedResource._free_native_ptr = self._real_free + + def _make_signer(self): + with open(os.path.join(self.data_dir, "es256_certs.pem"), "rb") as f: + certs = f.read() + with open(os.path.join(self.data_dir, "es256_private.key"), "rb") as f: + key = f.read() + return Signer.from_info(C2paSignerInfo( + b"es256", certs, key, b"http://timestamp.digicert.com")) + # _cleanup_resources: a failing _release() must not strand the handle def test_release_failure_still_frees_handle(self): - res = _CallbackHoldingResource() + res = self._CallbackHoldingResource() # _callback_cb is never set, so _release() raises AttributeError. res._activate(0xBBBB) @@ -6986,7 +6997,7 @@ def test_release_failure_still_frees_handle(self): self.assertIsNone(res._handle) def test_release_failure_is_logged(self): - res = _CallbackHoldingResource() + res = self._CallbackHoldingResource() res._activate(0xBBBB) with self.assertLogs('c2pa', level='ERROR') as captured: @@ -6999,7 +7010,7 @@ def test_release_failure_is_logged(self): # _activate guards def test_activate_rejects_null_handle(self): - res = _FakeHandleResource() + res = self._FakeHandleResource() with self.assertRaises(Error) as ctx: res._activate(None) @@ -7008,7 +7019,7 @@ def test_activate_rejects_null_handle(self): self.assertEqual(res._lifecycle_state, LifecycleState.UNINITIALIZED) def test_activate_rejects_double_activation(self): - res = _FakeHandleResource() + res = self._FakeHandleResource() res._activate(0x1111) with self.assertRaises(Error) as ctx: @@ -7021,7 +7032,7 @@ def test_activate_rejects_double_activation(self): self.assertEqual(self.freed, [0x1111]) def test_activate_rejects_reactivation_after_close(self): - res = _FakeHandleResource() + res = self._FakeHandleResource() res._activate(0x3333) res.close() self.freed.clear() @@ -7035,7 +7046,7 @@ def test_activate_rejects_reactivation_after_close(self): self.assertEqual(self.freed, []) def test_activate_does_not_mutate_on_rejection(self): - res = _FakeHandleResource() + res = self._FakeHandleResource() res._activate(0x5555) with self.assertRaises(Error): @@ -7047,7 +7058,7 @@ def test_activate_does_not_mutate_on_rejection(self): # _swap_handle def test_swap_handle_does_not_free_consumed_handle(self): - res = _FakeHandleResource() + res = self._FakeHandleResource() res._activate(0xAAA1) res._swap_handle(0xAAA2) @@ -7061,19 +7072,19 @@ def test_swap_handle_does_not_free_consumed_handle(self): self.assertEqual(self.freed, [0xAAA2]) def test_swap_handle_requires_active_resource(self): - uninitialized = _FakeHandleResource() + uninitialized = self._FakeHandleResource() with self.assertRaises(Error) as ctx: uninitialized._swap_handle(0x1) self.assertIn("not active", str(ctx.exception)) - closed = _FakeHandleResource() + closed = self._FakeHandleResource() closed._activate(0x2) closed.close() with self.assertRaises(Error): closed._swap_handle(0x3) def test_swap_handle_rejects_null_replacement(self): - res = _FakeHandleResource() + res = self._FakeHandleResource() res._activate(0x7777) with self.assertRaises(Error) as ctx: @@ -7109,10 +7120,10 @@ def _release(self): def test_wrap_native_handle_rejects_null(self): with self.assertRaises(Error): - _FakeHandleResource._wrap_native_handle(None) + self._FakeHandleResource._wrap_native_handle(None) def test_close_after_wrap_is_idempotent(self): - obj = _FakeHandleResource._wrap_native_handle(0xD00D) + obj = self._FakeHandleResource._wrap_native_handle(0xD00D) obj.close() obj.close() @@ -7122,14 +7133,14 @@ def test_close_after_wrap_is_idempotent(self): def test_every_construction_path_records_owner_pid(self): pid = os.getpid() - plain = _FakeHandleResource() + plain = self._FakeHandleResource() self.assertEqual(plain._owner_pid, pid) - activated = _FakeHandleResource() + activated = self._FakeHandleResource() activated._activate(0xA1) self.assertEqual(activated._owner_pid, pid) - wrapped = _FakeHandleResource._wrap_native_handle(0xA2) + wrapped = self._FakeHandleResource._wrap_native_handle(0xA2) self.assertEqual(wrapped._owner_pid, pid) # A swap keeps the original stamp: @@ -7145,7 +7156,7 @@ def test_activate_rejects_reserved_extra_attrs(self): ('_lifecycle_state', LifecycleState.CLOSED), ): with self.subTest(attr=attr): - res = _FakeHandleResource() + res = self._FakeHandleResource() stamp = res._owner_pid with self.assertRaises(Error) as ctx: @@ -7159,15 +7170,15 @@ def test_activate_rejects_reserved_extra_attrs(self): def test_wrap_native_handle_rejects_reserved_extra_attrs(self): with self.assertRaises(Error): - _FakeHandleResource._wrap_native_handle( + self._FakeHandleResource._wrap_native_handle( 0xB2, _owner_pid=os.getpid() + 1) def test_foreign_child_skips_free_for_wrapped_and_swapped(self): - wrapped = _FakeHandleResource._wrap_native_handle(0xC1) + wrapped = self._FakeHandleResource._wrap_native_handle(0xC1) wrapped._owner_pid = os.getpid() + 1 wrapped.close() - swapped = _FakeHandleResource() + swapped = self._FakeHandleResource() swapped._activate(0xC2) swapped._swap_handle(0xC3) swapped._owner_pid = os.getpid() + 1 @@ -7181,11 +7192,11 @@ def test_foreign_child_skips_free_for_wrapped_and_swapped(self): self.assertEqual(swapped._handle, 0xC3) def test_owning_process_frees_wrapped_and_swapped_exactly_once(self): - wrapped = _FakeHandleResource._wrap_native_handle(0xC4) + wrapped = self._FakeHandleResource._wrap_native_handle(0xC4) wrapped.close() wrapped.close() - swapped = _FakeHandleResource() + swapped = self._FakeHandleResource() swapped._activate(0xC5) swapped._swap_handle(0xC6) swapped.close() @@ -7195,24 +7206,24 @@ def test_owning_process_frees_wrapped_and_swapped_exactly_once(self): self.assertEqual(self._free_counts(), {0xC4: 1, 0xC6: 1}) def test_foreign_child_skips_release(self): - foreign = _ReleaseRecordingResource() + foreign = self._ReleaseRecordingResource() foreign._activate(0xD1) foreign._owner_pid = os.getpid() + 1 foreign.close() self.assertEqual(foreign.release_calls, 0) - owned = _ReleaseRecordingResource() + owned = self._ReleaseRecordingResource() owned._activate(0xD2) owned.close() self.assertEqual(owned.release_calls, 1) def test_consumed_resource_frees_nothing_in_either_process(self): - owned = _FakeHandleResource() + owned = self._FakeHandleResource() owned._activate(0xE1) owned._mark_consumed() owned.close() - foreign = _FakeHandleResource() + foreign = self._FakeHandleResource() foreign._activate(0xE2) foreign._mark_consumed() foreign._owner_pid = os.getpid() + 1 @@ -7220,6 +7231,83 @@ def test_consumed_resource_frees_nothing_in_either_process(self): self.assertEqual(self.freed, []) + def test_signer_init_rejects_null_pointer(self): + with self.assertRaises(Error): + Signer(None) + + def test_builder_from_archive_wraps_handle(self): + self._use_real_frees() + archive = io.BytesIO() + Builder({"claim_generator": "test", "format": "image/jpeg"}).to_archive( + archive) + + builder = Builder.from_archive(io.BytesIO(archive.getvalue())) + + self.assertTrue(builder.is_valid) + self.assertIsNone(builder._context) + self.assertFalse(builder._has_context_signer) + builder.close() + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) + + def test_context_build_failure_consumes_signer(self): + # c2pa_context_builder_set_signer takes ownership of the signer + # pointer immediately, so a later build failure must still leave the + # Signer consumed. Otherwise it holds a pointer the native side has + # already freed. + self._use_real_frees() + signer = self._make_signer() + real_build = c2pa_module._lib.c2pa_context_builder_build + c2pa_module._lib.c2pa_context_builder_build = lambda ptr: None + try: + with self.assertRaises(Error): + Context(signer=signer) + finally: + c2pa_module._lib.c2pa_context_builder_build = real_build + + self.assertIsNone(signer._handle, + "Signer still holds a pointer the native side freed") + self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) + + # Nothing left to free, so close() must be a no-op. + freed = [] + real_free = ManagedResource._free_native_ptr + ManagedResource._free_native_ptr = staticmethod(freed.append) + try: + signer.close() + finally: + ManagedResource._free_native_ptr = real_free + self.assertEqual(freed, []) + + def test_context_with_signer_consumes_it_on_success(self): + self._use_real_frees() + signer = self._make_signer() + + context = Context(signer=signer) + + self.assertTrue(context.is_valid) + self.assertIsNone(signer._handle) + self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) + self.assertTrue(context.has_signer) + context.close() + + def test_construction_failure_leaves_nothing_to_free(self): + # Activation happens after the null check, so a failed construction + # leaves no handle on the object for __del__ to find. + real_new = c2pa_module._lib.c2pa_context_new + c2pa_module._lib.c2pa_context_new = lambda: None + try: + with self.assertRaises(Error): + Context() + finally: + c2pa_module._lib.c2pa_context_new = real_new + + real_json = c2pa_module._lib.c2pa_builder_from_json + c2pa_module._lib.c2pa_builder_from_json = lambda j: None + try: + with self.assertRaises(Error): + Builder({"claim_generator": "test"}) + finally: + c2pa_module._lib.c2pa_builder_from_json = real_json class TestManagedResourceObjects(TestContextAPIs): """Tests native resource handling management when managed manually. @@ -7245,8 +7333,6 @@ def _make_archive(self, manifest=None): archive.seek(0) return archive - # Activation: every public constructor stamps the creating process - def test_settings_activation_paths(self): pid = os.getpid() for label, factory in ( @@ -7346,8 +7432,6 @@ def test_signer_activation_paths(self): self.addCleanup(callback_signer.close) self.assertEqual(callback_signer._owner_pid, os.getpid()) - # Swapping: the two public consume-and-return APIs - def test_builder_with_archive_swaps_the_handle(self): context = Context() self.addCleanup(context.close) @@ -7503,97 +7587,5 @@ def test_reader_with_fragment_null_return_consumes_self(self): self.assertEqual(freed, []) -class TestNativeHandleOwnership(unittest.TestCase): - """Ownership hand-offs between Python and the native library, driven by - fault injection at the FFI boundary.""" - - def setUp(self): - self.data_dir = os.path.join(os.path.dirname(__file__), "fixtures") - - def _make_signer(self): - with open(os.path.join(self.data_dir, "es256_certs.pem"), "rb") as f: - certs = f.read() - with open(os.path.join(self.data_dir, "es256_private.key"), "rb") as f: - key = f.read() - return Signer.from_info(C2paSignerInfo( - b"es256", certs, key, b"http://timestamp.digicert.com")) - - def test_signer_init_rejects_null_pointer(self): - with self.assertRaises(Error): - Signer(None) - - def test_builder_from_archive_wraps_handle(self): - archive = io.BytesIO() - Builder({"claim_generator": "test", "format": "image/jpeg"}).to_archive( - archive) - - builder = Builder.from_archive(io.BytesIO(archive.getvalue())) - - self.assertTrue(builder.is_valid) - self.assertIsNone(builder._context) - self.assertFalse(builder._has_context_signer) - builder.close() - self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) - - def test_context_build_failure_consumes_signer(self): - # c2pa_context_builder_set_signer takes ownership of the signer - # pointer immediately, so a later build failure must still leave the - # Signer consumed. Otherwise it holds a pointer the native side has - # already freed. - signer = self._make_signer() - real_build = c2pa_module._lib.c2pa_context_builder_build - c2pa_module._lib.c2pa_context_builder_build = lambda ptr: None - try: - with self.assertRaises(Error): - Context(signer=signer) - finally: - c2pa_module._lib.c2pa_context_builder_build = real_build - - self.assertIsNone(signer._handle, - "Signer still holds a pointer the native side freed") - self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) - - # Nothing left to free, so close() must be a no-op. - freed = [] - real_free = ManagedResource._free_native_ptr - ManagedResource._free_native_ptr = staticmethod(freed.append) - try: - signer.close() - finally: - ManagedResource._free_native_ptr = real_free - self.assertEqual(freed, []) - - def test_context_with_signer_consumes_it_on_success(self): - signer = self._make_signer() - - context = Context(signer=signer) - - self.assertTrue(context.is_valid) - self.assertIsNone(signer._handle) - self.assertEqual(signer._lifecycle_state, LifecycleState.CLOSED) - self.assertTrue(context.has_signer) - context.close() - - def test_construction_failure_leaves_nothing_to_free(self): - # A failed FFI construction must not leave a half-constructed object - # holding a handle. Activation happens after the check, so _handle is - # never set. - real_new = c2pa_module._lib.c2pa_context_new - c2pa_module._lib.c2pa_context_new = lambda: None - try: - with self.assertRaises(Error): - Context() - finally: - c2pa_module._lib.c2pa_context_new = real_new - - real_json = c2pa_module._lib.c2pa_builder_from_json - c2pa_module._lib.c2pa_builder_from_json = lambda j: None - try: - with self.assertRaises(Error): - Builder({"claim_generator": "test"}) - finally: - c2pa_module._lib.c2pa_builder_from_json = real_json - - if __name__ == '__main__': unittest.main(warnings='ignore') From 222b9a8b31454fd763c81f0a8345ad33456e8261 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:31:23 -0700 Subject: [PATCH 06/67] fix: Update tests --- src/c2pa/c2pa.py | 81 ++++++++++++------ tests/test_unit_tests.py | 178 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 226 insertions(+), 33 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 9ef2f4d3..dec8a8bc 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -237,7 +237,7 @@ class ManagedResource: Subclasses must: - Call `_activate(handle)` once the native pointer is created and - validated, which takes ownership of it and marks the resource ACTIVE. + validated, which takes ownership of it and marks the resource active. Never assign `self._handle` or `self._lifecycle_state` directly. - Call `_swap_handle(new_handle)` instead when an FFI call consumed the current handle and returned a replacement. @@ -246,10 +246,22 @@ class ManagedResource: - Override `_release()` to free class-specific resources (streams, caches, callbacks, etc.), called before the native pointer is freed. + - Override `_init_attrs()` to set the class's own attributes to their + defaults, and call it from `__init__`. `_wrap_native_handle` calls it + too, so an instance built around an existing handle is never missing + the attributes the rest of the class reads. The native pointer is freed automatically via `_free_native_ptr`. """ + def _init_attrs(self): + """Set this class's own attributes to their defaults. + + Called by __init__ and by _wrap_native_handle, which bypasses + __init__. Keeping the defaults here means an instance built around an + existing handle is never missing what the rest of the class reads. + """ + def __init__(self): self._lifecycle_state = LifecycleState.UNINITIALIZED self._handle = None @@ -371,14 +383,17 @@ def _wrap_native_handle(cls, handle, **extra_attrs): """Build a brand-new instance around an already-valid, already-owned native handle, bypassing __init__ entirely. - Because __init__ is bypassed, every attribute the subclass's - `_release()` reads must be passed in `extra_attrs`. - The lifecycle attributes are still off limits there, - and the instance is stamped with the creating process either way. + __init__ is bypassed, so `_init_attrs()` supplies the class's own + defaults; `extra_attrs` is only for overriding them. The lifecycle + attributes are off limits there, and the instance is stamped with the + creating process either way. + + Ownership of `handle` only transfers once this returns. If it raises, + the caller still owns the pointer and must free it. Args: handle: Non-null native pointer to take ownership of - **extra_attrs: Instance attributes to set before activating + **extra_attrs: Instance attributes overriding the defaults Raises: C2paError: If the handle is null, or extra_attrs names a @@ -387,6 +402,7 @@ def _wrap_native_handle(cls, handle, **extra_attrs): obj = object.__new__(cls) # Stamps _owner_pid, which the fork guard relies on. ManagedResource.__init__(obj) + obj._init_attrs() obj._activate(handle, **extra_attrs) return obj @@ -1549,8 +1565,7 @@ def __init__( C2paError: If creation fails """ super().__init__() - self._has_signer = False - self._signer_callback_cb = None + self._init_attrs() if settings is None and signer is None: # Simple default context @@ -1614,6 +1629,10 @@ def __init__( pass raise + def _init_attrs(self): + self._has_signer = False + self._signer_callback_cb = None + def _release(self): """Release Context-specific resources.""" self._signer_callback_cb = None @@ -2262,20 +2281,7 @@ def __init__( contain invalid UTF-8 characters """ super().__init__() - - self._own_stream = None - - # This is used to keep track of a file - # we may have opened ourselves, and that we need to close later - self._backing_file = None - - # Caches for manifest JSON string and parsed data. - # These are invalidated when with_fragment() is called, because each - # new BMFF fragment can refine or update the manifest content as the - # reader progressively builds its understanding of the fragmented stream. - # They are also cleared on close() to release memory. - self._manifest_json_str_cache = None - self._manifest_data_cache = None + self._init_attrs() self._context = context @@ -2460,6 +2466,22 @@ def _init_from_context(self, context, format_or_path, self._close_streams() raise + def _init_attrs(self): + self._own_stream = None + + # Tracks a file we opened ourselves and must close later. + self._backing_file = None + + # Caches for manifest JSON string and parsed data. + # These are invalidated when with_fragment() is called, because each + # new BMFF fragment can refine or update the manifest content as the + # reader progressively builds its understanding of the fragmented + # stream. They are also cleared on close() to release memory. + self._manifest_json_str_cache = None + self._manifest_data_cache = None + + self._context = None + def _close_streams(self): """Close owned stream and backing file if present.""" if getattr(self, '_own_stream', None): @@ -3129,8 +3151,14 @@ def from_archive( "Failed to create builder from archive" ) - return cls._wrap_native_handle( - handle, _context=None, _has_context_signer=False) + try: + # A builder from an archive carries no context, which is what + # _init_attrs() already defaults to. + return cls._wrap_native_handle(handle) + except Exception: + # No instance took ownership, so the handle is still ours. + ManagedResource._free_native_ptr(handle) + raise finally: stream_obj.close() @@ -3164,6 +3192,7 @@ def __init__( C2paError.Json: If the manifest JSON cannot be serialized """ super().__init__() + self._init_attrs() self._context = context self._has_context_signer = ( @@ -3221,6 +3250,10 @@ def _init_from_context(self, context, json_str): self._activate(new_ptr) + def _init_attrs(self): + self._context = None + self._has_context_signer = False + def set_no_embed(self): """Set the no-embed flag. diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index fd7b3669..180447b0 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -11,9 +11,12 @@ # specific language governing permissions and limitations under # each license. +import gc +import inspect import os import io import json +import re import unittest import ctypes import warnings @@ -7309,12 +7312,29 @@ def test_construction_failure_leaves_nothing_to_free(self): finally: c2pa_module._lib.c2pa_builder_from_json = real_json +def _ptr_addr(ptr): + """Address a ctypes pointer points at, or None for a null pointer. + + ctypes pointers compare by identity, not by value: two pointer objects + for the same address are unequal. Compare addresses instead. + """ + if not ptr: + return None + return ctypes.cast(ptr, ctypes.c_void_p).value + + class TestManagedResourceObjects(TestContextAPIs): """Tests native resource handling management when managed manually. """ def _instrument_frees(self): - """Record frees instead of performing them, and restore on teardown.""" + """Record frees instead of performing them, and restore on teardown. + + This patches the base class, so every ManagedResource freed while the + patch is installed lands in the list, including objects the garbage + collector reclaims mid-test. Ask _free_count() about one handle rather + than asserting on the length of the list. + """ freed = [] real_free = ManagedResource._free_native_ptr ManagedResource._free_native_ptr = staticmethod(freed.append) @@ -7323,6 +7343,12 @@ def _instrument_frees(self): ManagedResource, '_free_native_ptr', real_free)) return freed + def _free_count(self, freed, handle): + """How many times `handle` was freed, ignoring unrelated frees.""" + target = _ptr_addr(handle) + self.assertIsNotNone(target, "cannot count frees of a null handle") + return sum(1 for ptr in freed if _ptr_addr(ptr) == target) + def _make_archive(self, manifest=None): archive = io.BytesIO() builder = Builder(manifest or self.test_manifest) @@ -7476,8 +7502,6 @@ def test_swapped_builder_is_freed_exactly_once(self): self.addCleanup(context.close) builder = Builder(self.test_manifest, context=context) original_handle = builder._handle - # The helper closes a temporary Builder, - # whose free would otherwise be counted here. archive = self._make_archive() # Instrument across the swap so a free of the consumed pointer is recorded. @@ -7485,15 +7509,16 @@ def test_swapped_builder_is_freed_exactly_once(self): builder.with_archive(archive) swapped_handle = builder._handle - self.assertEqual(freed, [], "the swap freed the consumed pointer") + self.assertEqual(self._free_count(freed, original_handle), 0, + "the swap freed the consumed pointer") builder.close() builder.close() # Only the replacement is ours to free: the original was consumed by # the FFI call that returned it. - self.assertEqual(freed, [swapped_handle]) - self.assertNotIn(original_handle, freed) + self.assertEqual(self._free_count(freed, swapped_handle), 1) + self.assertEqual(self._free_count(freed, original_handle), 0) def test_repeated_swaps_on_one_builder(self): # Each with_archive consumes the handle the previous one returned, so @@ -7533,18 +7558,22 @@ def test_context_consumes_signer_but_not_settings(self): def test_consumed_signer_close_frees_nothing(self): signer = self._ctx_make_signer() + # Captured before the context consumes it: close() nulls the handle, + # so afterwards there is no pointer left to identify the free by. + signer_handle = signer._handle context = Context(signer=signer) self.addCleanup(context.close) freed = self._instrument_frees() signer.close() - self.assertEqual(freed, [], + self.assertEqual(self._free_count(freed, signer_handle), 0, "closing a consumed Signer freed a pointer the " "context now owns") def test_builder_with_archive_null_return_consumes_self(self): builder = Builder(self.test_manifest) + consumed_handle = builder._handle real_call = c2pa_module._lib.c2pa_builder_with_archive c2pa_module._lib.c2pa_builder_with_archive = lambda b, s: None try: @@ -7560,13 +7589,15 @@ def test_builder_with_archive_null_return_consumes_self(self): freed = self._instrument_frees() builder.close() - self.assertEqual(freed, []) + self.assertEqual(self._free_count(freed, consumed_handle), 0, + "close() freed a handle the FFI already consumed") def test_reader_with_fragment_null_return_consumes_self(self): init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") with open(init_path, "rb") as init: reader = Reader("video/mp4", init) + consumed_handle = reader._handle real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( @@ -7584,7 +7615,136 @@ def test_reader_with_fragment_null_return_consumes_self(self): freed = self._instrument_frees() reader.close() - self.assertEqual(freed, []) + self.assertEqual(self._free_count(freed, consumed_handle), 0, + "close() freed a handle the FFI already consumed") + + # Backfilling a pointer minted by a direct FFI call. Builder.from_archive + # is the only production caller of _wrap_native_handle, so these are the + # only tests that drive the primitive as the generic entry point it is. + + def _raw_builder_handle(self): + manifest = json.dumps( + {"claim_generator": "raw_ffi_test", "format": "image/jpeg"} + ).encode("utf-8") + handle = c2pa_module._lib.c2pa_builder_from_json(manifest) + self.assertTrue(handle, "the FFI did not return a builder pointer") + return handle + + def test_wrap_raw_ffi_builder_pointer(self): + builder = Builder._wrap_native_handle( + self._raw_builder_handle(), + _context=None, _has_context_signer=False) + + self.assertTrue(builder.is_valid) + self.assertEqual(builder._lifecycle_state, LifecycleState.ACTIVE) + self.assertEqual(builder._owner_pid, os.getpid()) + + archive = io.BytesIO() + builder.to_archive(archive) + self.assertTrue(archive.getvalue()) + + builder.close() + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(builder._handle) + + def test_wrap_raw_ffi_settings_pointer(self): + handle = c2pa_module._lib.c2pa_settings_new() + self.assertTrue(handle) + + settings = Settings._wrap_native_handle(handle) + try: + self.assertTrue(settings.is_valid) + self.assertEqual(settings._owner_pid, os.getpid()) + settings.set("version_major", "1") + finally: + settings.close() + + def test_wrap_raw_ffi_context_pointer(self): + handle = c2pa_module._lib.c2pa_context_new() + self.assertTrue(handle) + + context = Context._wrap_native_handle( + handle, _has_signer=False, _signer_callback_cb=None) + try: + self.assertTrue(context.is_valid) + self.assertEqual(context._owner_pid, os.getpid()) + # Handing the wrapped pointer back to the FFI proves it is live. + builder = Builder(self.test_manifest, context=context) + self.assertTrue(builder.is_valid) + builder.close() + finally: + context.close() + + def test_wrapped_raw_pointer_freed_exactly_once(self): + handle = self._raw_builder_handle() + freed = self._instrument_frees() + + builder = Builder._wrap_native_handle( + handle, _context=None, _has_context_signer=False) + builder.close() + builder.close() + del builder + gc.collect() + + self.assertEqual(self._free_count(freed, handle), 1, + "wrapped handle not freed exactly once") + + def test_wrap_supplies_defaults_without_extra_attrs(self): + # _init_attrs() runs on the wrap path, so a caller that passes no + # extra_attrs still gets an instance the rest of the class can read. + builder = Builder._wrap_native_handle(self._raw_builder_handle()) + self.addCleanup(builder.close) + + self.assertIsNone(builder._context) + self.assertFalse(builder._has_context_signer) + + def test_wrap_extra_attrs_override_defaults(self): + context = Context() + self.addCleanup(context.close) + + builder = Builder._wrap_native_handle( + self._raw_builder_handle(), _context=context) + self.addCleanup(builder.close) + + self.assertIs(builder._context, context) + # Untouched by the override, so still the default. + self.assertFalse(builder._has_context_signer) + + def test_init_attrs_covers_what_init_sets(self): + # Anything __init__ sets but _init_attrs() misses is absent on a + # wrapped instance, which is the trap _init_attrs() exists to close. + for cls in (Builder, Context, Reader): + with self.subTest(cls=cls.__name__): + defaulted = set(re.findall( + r"self\.(_[a-z][a-z0-9_]*)\s*=", + inspect.getsource(cls._init_attrs))) + assigned = set(re.findall( + r"self\.(_[a-z][a-z0-9_]*)\s*=", + inspect.getsource(cls.__init__))) + self.assertEqual( + assigned - defaulted, set(), + f"{cls.__name__}.__init__ sets attributes that " + f"_init_attrs() does not default") + + def test_from_archive_frees_handle_when_wrap_fails(self): + # The wrap raising means no Python object took ownership, so + # from_archive still holds the handle and has to free it. + archive = self._make_archive() # closes a Builder; keep it off the count + freed = self._instrument_frees() + real_wrap = Builder._wrap_native_handle + + def _boom(*args, **kwargs): + raise Error("wrap failed") + + Builder._wrap_native_handle = _boom + try: + with self.assertRaises(Error): + Builder.from_archive(archive) + finally: + Builder._wrap_native_handle = real_wrap + + self.assertEqual(len(freed), 1, + "from_archive leaked the handle when the wrap failed") if __name__ == '__main__': From 3f93e7e85c0c84e3d7801b5012607faf02cac913 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:41:58 -0700 Subject: [PATCH 07/67] fix: Improve pointers handling --- src/c2pa/c2pa.py | 26 +++++++++++++---- tests/test_unit_tests.py | 60 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index dec8a8bc..6de29389 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -314,9 +314,8 @@ def _activate(self, handle, **extra_attrs): Only an uninitialized resource can be activated, so a handle can never be activated twice and a closed resource can never be reopened. - Any attribute the subclass's `_release()` reads must be passed in - `extra_attrs` when __init__ did not already set it, otherwise - `_release()` raises during cleanup. + `extra_attrs` overrides the defaults `_init_attrs()` already supplied, + so it is only for values that differ from them. `extra_attrs` cannot carry the lifecycle attributes themselves. `_owner_pid` in particular records the process that created the @@ -1630,6 +1629,7 @@ def __init__( raise def _init_attrs(self): + super()._init_attrs() self._has_signer = False self._signer_callback_cb = None @@ -2467,6 +2467,7 @@ def _init_from_context(self, context, format_or_path, raise def _init_attrs(self): + super()._init_attrs() self._own_stream = None # Tracks a file we opened ourselves and must close later. @@ -2484,14 +2485,14 @@ def _init_attrs(self): def _close_streams(self): """Close owned stream and backing file if present.""" - if getattr(self, '_own_stream', None): + if self._own_stream: try: self._own_stream.close() except Exception: logger.error("Failed to close Reader stream") finally: self._own_stream = None - if getattr(self, '_backing_file', None): + if self._backing_file: try: self._backing_file.close() except Exception: @@ -3015,11 +3016,18 @@ def __init__(self, signer_ptr: ctypes.POINTER(C2paSigner)): C2paError: If the signer pointer is invalid """ super().__init__() + self._init_attrs() if not signer_ptr: raise C2paError("Invalid signer pointer: pointer is null") - self._activate(signer_ptr, _callback_cb=None) + self._activate(signer_ptr) + + def _init_attrs(self): + super()._init_attrs() + # from_callback() replaces this with the real callback, which has to + # outlive the signer that calls it. + self._callback_cb = None def _release(self): """Release Signer-specific resources (callback reference).""" @@ -3251,9 +3259,15 @@ def _init_from_context(self, context, json_str): self._activate(new_ptr) def _init_attrs(self): + super()._init_attrs() self._context = None self._has_context_signer = False + def _release(self): + """Release the Builder's reference to its Context.""" + # The Context is not ours to close, only to stop pinning. + self._context = None + def set_no_embed(self): """Set the no-embed flag. diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 180447b0..53156af0 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -7713,7 +7713,7 @@ def test_wrap_extra_attrs_override_defaults(self): def test_init_attrs_covers_what_init_sets(self): # Anything __init__ sets but _init_attrs() misses is absent on a # wrapped instance, which is the trap _init_attrs() exists to close. - for cls in (Builder, Context, Reader): + for cls in (Builder, Context, Reader, Signer): with self.subTest(cls=cls.__name__): defaulted = set(re.findall( r"self\.(_[a-z][a-z0-9_]*)\s*=", @@ -7726,6 +7726,64 @@ def test_init_attrs_covers_what_init_sets(self): f"{cls.__name__}.__init__ sets attributes that " f"_init_attrs() does not default") + def test_init_attrs_overrides_chain_to_super(self): + # A subclass of these would silently lose the parent's defaults if + # the chain were broken. + for cls in (Builder, Context, Reader, Signer): + with self.subTest(cls=cls.__name__): + self.assertIn( + "super()._init_attrs()", + inspect.getsource(cls._init_attrs), + f"{cls.__name__}._init_attrs() does not chain to super()") + + def test_wrap_raw_ffi_signer_pointer(self): + # Signer._release() reads _callback_cb, so a wrap that skipped the + # defaults would fail during cleanup rather than at the wrap. + freed = self._instrument_frees() + + signer = Signer._wrap_native_handle(0xABCD) + self.assertIsNone(signer._callback_cb) + self.assertEqual(signer._owner_pid, os.getpid()) + + # Cleanup swallows a failing _release(), so the error log is the only + # way to see one. + with self.assertNoLogs("c2pa", level="ERROR"): + signer.close() + + self.assertEqual(freed, [0xABCD]) + + def test_signer_release_clears_callback(self): + signer = self._ctx_make_callback_signer() + self.assertIsNotNone(signer._callback_cb) + + signer.close() + + self.assertIsNone(signer._callback_cb) + + def test_builder_release_clears_context(self): + context = Context() + self.addCleanup(context.close) + builder = Builder(self.test_manifest, context=context) + + builder.close() + + self.assertIsNone(builder._context, + "closed Builder still pins its Context") + # The Builder does not own the Context, so it must not close it. + self.assertTrue(context.is_valid) + + def test_reader_close_closes_backing_file(self): + # _close_streams reads the attrs _init_attrs() defaults, so this is + # the regression guard for reading them directly. + reader = Reader(DEFAULT_TEST_FILE) + backing_file = reader._backing_file + self.assertIsNotNone(backing_file) + + reader.close() + + self.assertTrue(backing_file.closed, "Reader left its file open") + self.assertIsNone(reader._backing_file) + def test_from_archive_frees_handle_when_wrap_fails(self): # The wrap raising means no Python object took ownership, so # from_archive still holds the handle and has to free it. From 97fe07e784709a3c1859b7c23e664bb1ee4671ec Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:57:57 -0700 Subject: [PATCH 08/67] fix: Improve pointers handling 2 --- src/c2pa/c2pa.py | 39 +++++++++--- tests/test_unit_tests.py | 128 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 8 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 6de29389..da33b949 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -299,9 +299,26 @@ def _release(self): def _mark_consumed(self): """Mark as consumed by an FFI call that took ownership - of native resources e.g. pointers. This means we should not - call clean-up here anymore, and leave it to the new owner. + of native resources e.g. pointers. The new owner frees the handle, + but this class's own resources are still ours to let go of, and + marking the resource closed means _cleanup_resources will not do it + later. """ + if is_foreign_process(self): + self._handle = None + self._lifecycle_state = LifecycleState.CLOSED + return + + # Callers raise straight after consuming, so a failing _release() + # here would mask the error they are reporting. + try: + self._release() + except Exception: + logger.error( + "Failed to release %s resources", + type(self).__name__, + exc_info=True, + ) self._handle = None self._lifecycle_state = LifecycleState.CLOSED @@ -1986,6 +2003,11 @@ def close(self): if self._closed: return if is_foreign_process(self): + # Unlike ManagedResource, which leaves a child's copy active + # because the parent still owns the pointer, a Stream's callbacks + # are bound to the parent's objects. + # The child's copy can never be used, so mark it closed + # and let the parent free it. self._closed = True self._initialized = False return @@ -2501,7 +2523,13 @@ def _close_streams(self): self._backing_file = None def _release(self): - """Release Reader-specific resources (stream, backing file).""" + """Release Reader-specific resources (caches, stream, backing file). + + Every teardown path runs this, including _mark_consumed(), which + close() never gets a chance to follow. + """ + self._manifest_json_str_cache = None + self._manifest_data_cache = None self._close_streams() def _get_cached_manifest_data(self) -> Optional[dict]: @@ -2583,11 +2611,6 @@ def with_fragment(self, format: str, stream, return self - def close(self): - """Release the reader resources.""" - self._manifest_json_str_cache = None - self._manifest_data_cache = None - super().close() def json(self) -> str: """Get the manifest store as a JSON string. diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 53156af0..fadc7bb7 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -7234,6 +7234,41 @@ def test_consumed_resource_frees_nothing_in_either_process(self): self.assertEqual(self.freed, []) + # Consuming a handle hands the native pointer to a new owner, + # but the Python-side resources are still ours to let go of. + + def test_mark_consumed_releases_python_resources(self): + res = self._ReleaseRecordingResource() + res._activate(0xF1) + + res._mark_consumed() + + self.assertEqual(res.release_calls, 1) + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(res._handle) + self.assertEqual(self.freed, []) + + def test_mark_consumed_swallows_failing_release(self): + res = self._CallbackHoldingResource() + res._activate(0xF2) + + with self.assertLogs("c2pa", level="ERROR"): + res._mark_consumed() + + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(res._handle) + + def test_mark_consumed_in_foreign_process_skips_release(self): + res = self._ReleaseRecordingResource() + res._activate(0xF3) + res._owner_pid = os.getpid() + 1 + + res._mark_consumed() + + self.assertEqual(res.release_calls, 0) + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(res._handle) + def test_signer_init_rejects_null_pointer(self): with self.assertRaises(Error): Signer(None) @@ -7772,10 +7807,64 @@ def test_builder_release_clears_context(self): # The Builder does not own the Context, so it must not close it. self.assertTrue(context.is_valid) + def test_consumed_reader_closes_backing_file(self): + # A failed with_fragment consumes the reader. + # # Reader(path) opened the backing file itself, + # so nothing else will ever close it. + reader = Reader(DEFAULT_TEST_FILE) + backing_file = reader._backing_file + self.assertFalse(backing_file.closed) + + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = ( + lambda r, f, s, frag: None) + try: + with open(DEFAULT_TEST_FILE, "rb") as main, \ + open(DEFAULT_TEST_FILE, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("image/jpeg", main, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + + self.assertTrue(backing_file.closed, + "consumed Reader leaked its backing file") + + def test_consumed_builder_releases_context(self): + context = Context() + self.addCleanup(context.close) + builder = Builder(self.test_manifest, context=context) + archive = self._make_archive() + + real_call = c2pa_module._lib.c2pa_builder_with_archive + c2pa_module._lib.c2pa_builder_with_archive = lambda b, s: None + try: + with self.assertRaises(Error): + builder.with_archive(archive) + finally: + c2pa_module._lib.c2pa_builder_with_archive = real_call + + self.assertIsNone(builder._context, + "consumed Builder still pins its Context") + self.assertTrue(context.is_valid) + + def test_context_takes_callback_before_consuming_signer(self): + # Consuming the signer releases its callback reference, + # so the Context has to take it first or the callback dies with the signer. + signer = self._ctx_make_callback_signer() + callback = signer._callback_cb + self.assertIsNotNone(callback) + + context = Context(signer=signer) + self.addCleanup(context.close) + + self.assertIs(context._signer_callback_cb, callback) + self.assertIsNone(signer._callback_cb) + def test_reader_close_closes_backing_file(self): # _close_streams reads the attrs _init_attrs() defaults, so this is # the regression guard for reading them directly. reader = Reader(DEFAULT_TEST_FILE) + reader.json() backing_file = reader._backing_file self.assertIsNotNone(backing_file) @@ -7783,6 +7872,45 @@ def test_reader_close_closes_backing_file(self): self.assertTrue(backing_file.closed, "Reader left its file open") self.assertIsNone(reader._backing_file) + self.assertIsNone(reader._manifest_json_str_cache) + self.assertIsNone(reader._manifest_data_cache) + + def test_consumed_reader_clears_caches(self): + # Consuming marks the reader closed, so close() will not run later. + # Anything cleanup owes the object has to happen at consume time. + reader = Reader(DEFAULT_TEST_FILE) + reader.json() + self.assertIsNotNone(reader._manifest_json_str_cache) + + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = ( + lambda r, f, s, frag: None) + try: + with open(DEFAULT_TEST_FILE, "rb") as main, \ + open(DEFAULT_TEST_FILE, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("image/jpeg", main, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + + self.assertIsNone(reader._manifest_json_str_cache, + "consumed Reader kept its manifest cache") + self.assertIsNone(reader._manifest_data_cache) + + def test_reader_del_clears_caches(self): + # __del__ goes through _cleanup_resources, not close(), so cache + # clearing has to live somewhere both paths reach. + reader = Reader(DEFAULT_TEST_FILE) + reader.json() + self.assertIsNotNone(reader._manifest_json_str_cache) + + # __del__ runs _cleanup_resources directly, so drive that rather than + # dropping the reference: the assertions need the object afterwards. + reader._cleanup_resources() + + self.assertIsNone(reader._manifest_json_str_cache, + "cleanup left the manifest cache alive") + self.assertIsNone(reader._manifest_data_cache) def test_from_archive_frees_handle_when_wrap_fails(self): # The wrap raising means no Python object took ownership, so From d95a9d7f7f5c594482b2701577acea0f86c0a288 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:17:21 -0700 Subject: [PATCH 09/67] fix: The refactor --- src/c2pa/c2pa.py | 42 ++++------------------ tests/test_unit_tests.py | 77 +++++++++++----------------------------- 2 files changed, 28 insertions(+), 91 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index da33b949..26a6913f 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -224,12 +224,6 @@ class LifecycleState(enum.IntEnum): CLOSED = 2 -# Attributes that make up the lifecycle state itself, which _activate sets -# from its own arguments and must never accept from a caller's extra_attrs. -_RESERVED_ACTIVATION_ATTRS = frozenset( - {'_handle', '_lifecycle_state', '_owner_pid'}) - - class ManagedResource: """Base class for objects that hold a native (C FFI) resource. This is an internal base class that provides lifecycle management @@ -323,30 +317,19 @@ def _mark_consumed(self): self._handle = None self._lifecycle_state = LifecycleState.CLOSED - def _activate(self, handle, **extra_attrs): - """Attach a native handle (and any extra instance attrs) to self - and mark it active. Attaching activates it. + def _activate(self, handle): + """Attach a native handle to self and mark it active. Ownership of `handle` transfers here: this object frees it on close. Only an uninitialized resource can be activated, so a handle can never be activated twice and a closed resource can never be reopened. - `extra_attrs` overrides the defaults `_init_attrs()` already supplied, - so it is only for values that differ from them. - - `extra_attrs` cannot carry the lifecycle attributes themselves. - `_owner_pid` in particular records the process that created the - handle, and overwriting it would let a forked child free a pointer - its parent still owns. - Args: handle: Non-null native pointer to take ownership of - **extra_attrs: Instance attributes to set before activating Raises: C2paError: If the handle is null, - the resource is not uninitialized, - or extra_attrs names a lifecycle attribute + or the resource is not uninitialized """ name = type(self).__name__ # A rejected activation must leave the object as it was. @@ -356,14 +339,7 @@ def _activate(self, handle, **extra_attrs): raise C2paError( f"{name}: already activated " f"({self._lifecycle_state.name})") - reserved = _RESERVED_ACTIVATION_ATTRS.intersection(extra_attrs) - if reserved: - raise C2paError( - f"{name}: cannot set lifecycle attributes via extra_attrs: " - f"{', '.join(sorted(reserved))}") - for attr, value in extra_attrs.items(): - setattr(self, attr, value) self._handle = handle self._lifecycle_state = LifecycleState.ACTIVE @@ -395,31 +371,27 @@ def _swap_handle(self, new_handle): self._handle = new_handle @classmethod - def _wrap_native_handle(cls, handle, **extra_attrs): + def _wrap_native_handle(cls, handle): """Build a brand-new instance around an already-valid, already-owned native handle, bypassing __init__ entirely. __init__ is bypassed, so `_init_attrs()` supplies the class's own - defaults; `extra_attrs` is only for overriding them. The lifecycle - attributes are off limits there, and the instance is stamped with the - creating process either way. + defaults and the instance is stamped with the creating process. Ownership of `handle` only transfers once this returns. If it raises, the caller still owns the pointer and must free it. Args: handle: Non-null native pointer to take ownership of - **extra_attrs: Instance attributes overriding the defaults Raises: - C2paError: If the handle is null, or extra_attrs names a - lifecycle attribute + C2paError: If the handle is null """ obj = object.__new__(cls) # Stamps _owner_pid, which the fork guard relies on. ManagedResource.__init__(obj) obj._init_attrs() - obj._activate(handle, **extra_attrs) + obj._activate(handle) return obj def _cleanup_resources(self): diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index fadc7bb7..4824e377 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -6942,8 +6942,8 @@ class _FakeHandleResource(ManagedResource): """Concrete subclass with no resources of its own.""" class _CallbackHoldingResource(ManagedResource): - """Mimics Signer: its _release() reads an attribute that must have - been supplied by __init__ or by _activate's extra_attrs.""" + """Mimics Signer: its _release() reads an attribute that _init_attrs() + is responsible for defaulting.""" def _release(self): if self._callback_cb: @@ -7053,10 +7053,11 @@ def test_activate_does_not_mutate_on_rejection(self): res._activate(0x5555) with self.assertRaises(Error): - res._activate(0x6666, _extra='should not be set') + res._activate(0x6666) - self.assertFalse(hasattr(res, '_extra'), - "rejected activation still set extra_attrs") + self.assertEqual(res._handle, 0x5555, + "rejected activation replaced the handle") + self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) # _swap_handle @@ -7099,27 +7100,31 @@ def test_swap_handle_rejects_null_replacement(self): # _wrap_native_handle - def test_wrap_native_handle_sets_extra_attrs_and_bypasses_init(self): + def test_wrap_native_handle_bypasses_init(self): seen = [] class Probe(ManagedResource): def __init__(self): raise AssertionError("__init__ must be bypassed") + def _init_attrs(self): + super()._init_attrs() + self._tag = 'from _init_attrs' + def _release(self): seen.append(self._tag) - obj = Probe._wrap_native_handle(0xC0DE, _tag='from extra_attrs') + obj = Probe._wrap_native_handle(0xC0DE) - self.assertEqual(obj._tag, 'from extra_attrs') + self.assertEqual(obj._tag, 'from _init_attrs') self.assertEqual(obj._lifecycle_state, LifecycleState.ACTIVE) self.assertTrue(obj.is_valid) # ManagedResource.__init__ still ran, so fork-safety is intact. self.assertTrue(hasattr(obj, '_owner_pid')) obj.close() - self.assertEqual(seen, ['from extra_attrs'], - "_release() could not see the extra attrs") + self.assertEqual(seen, ['from _init_attrs'], + "_release() could not see the class's own attrs") def test_wrap_native_handle_rejects_null(self): with self.assertRaises(Error): @@ -7152,30 +7157,6 @@ def test_every_construction_path_records_owner_pid(self): wrapped._swap_handle(0xA3) self.assertEqual(wrapped._owner_pid, pid) - def test_activate_rejects_reserved_extra_attrs(self): - for attr, value in ( - ('_owner_pid', os.getpid() + 1), - ('_handle', 0xDEAD), - ('_lifecycle_state', LifecycleState.CLOSED), - ): - with self.subTest(attr=attr): - res = self._FakeHandleResource() - stamp = res._owner_pid - - with self.assertRaises(Error) as ctx: - res._activate(0xB1, **{attr: value}) - - self.assertIn(attr, str(ctx.exception)) - self.assertEqual(res._owner_pid, stamp) - self.assertEqual( - res._lifecycle_state, LifecycleState.UNINITIALIZED) - self.assertIsNone(res._handle) - - def test_wrap_native_handle_rejects_reserved_extra_attrs(self): - with self.assertRaises(Error): - self._FakeHandleResource._wrap_native_handle( - 0xB2, _owner_pid=os.getpid() + 1) - def test_foreign_child_skips_free_for_wrapped_and_swapped(self): wrapped = self._FakeHandleResource._wrap_native_handle(0xC1) wrapped._owner_pid = os.getpid() + 1 @@ -7666,9 +7647,7 @@ def _raw_builder_handle(self): return handle def test_wrap_raw_ffi_builder_pointer(self): - builder = Builder._wrap_native_handle( - self._raw_builder_handle(), - _context=None, _has_context_signer=False) + builder = Builder._wrap_native_handle(self._raw_builder_handle()) self.assertTrue(builder.is_valid) self.assertEqual(builder._lifecycle_state, LifecycleState.ACTIVE) @@ -7698,8 +7677,7 @@ def test_wrap_raw_ffi_context_pointer(self): handle = c2pa_module._lib.c2pa_context_new() self.assertTrue(handle) - context = Context._wrap_native_handle( - handle, _has_signer=False, _signer_callback_cb=None) + context = Context._wrap_native_handle(handle) try: self.assertTrue(context.is_valid) self.assertEqual(context._owner_pid, os.getpid()) @@ -7714,8 +7692,7 @@ def test_wrapped_raw_pointer_freed_exactly_once(self): handle = self._raw_builder_handle() freed = self._instrument_frees() - builder = Builder._wrap_native_handle( - handle, _context=None, _has_context_signer=False) + builder = Builder._wrap_native_handle(handle) builder.close() builder.close() del builder @@ -7724,27 +7701,15 @@ def test_wrapped_raw_pointer_freed_exactly_once(self): self.assertEqual(self._free_count(freed, handle), 1, "wrapped handle not freed exactly once") - def test_wrap_supplies_defaults_without_extra_attrs(self): - # _init_attrs() runs on the wrap path, so a caller that passes no - # extra_attrs still gets an instance the rest of the class can read. + def test_wrap_supplies_defaults(self): + # _init_attrs() runs on the wrap path, so an instance built around a + # raw handle still has everything the rest of the class reads. builder = Builder._wrap_native_handle(self._raw_builder_handle()) self.addCleanup(builder.close) self.assertIsNone(builder._context) self.assertFalse(builder._has_context_signer) - def test_wrap_extra_attrs_override_defaults(self): - context = Context() - self.addCleanup(context.close) - - builder = Builder._wrap_native_handle( - self._raw_builder_handle(), _context=context) - self.addCleanup(builder.close) - - self.assertIs(builder._context, context) - # Untouched by the override, so still the default. - self.assertFalse(builder._has_context_signer) - def test_init_attrs_covers_what_init_sets(self): # Anything __init__ sets but _init_attrs() misses is absent on a # wrapped instance, which is the trap _init_attrs() exists to close. From 0309a921b3df9d76eb7e7bfd97f051031a675bc3 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:19:45 -0700 Subject: [PATCH 10/67] fix: Improve pointers handling 4 --- pr294-finishing-plan.md | 121 +++++++++++++++++++++++++++++++++++++++ src/c2pa/c2pa.py | 49 ++++++++++------ tests/test_unit_tests.py | 56 +++++++++++++++++- 3 files changed, 207 insertions(+), 19 deletions(-) create mode 100644 pr294-finishing-plan.md diff --git a/pr294-finishing-plan.md b/pr294-finishing-plan.md new file mode 100644 index 00000000..3579c47c --- /dev/null +++ b/pr294-finishing-plan.md @@ -0,0 +1,121 @@ +# PR #294 — Finishing plan: native-handle lifecycle extension surface + +**Goal:** take #294 from WIP to mergeable and reviewer-proof. The code is +mechanically sound; the work left is *committing to the extension contract* and +writing it down where it can be enforced. No functional rewrite required. + +--- + +## Decision to record first: the extension model + +Downstream libraries should be able to wrap their own FFI pointers in the +managed lifecycle, **without** the project promising a stable public API. + +Chosen: **single-underscore, subclass-facing (protected).** Keep the names as +they are (`_activate`, `_swap_handle`, `_mark_consumed`, `_wrap_native_handle`, +`_init_attrs`). + +Why this level and not the alternatives — state this in the PR so it isn't +re-litigated in review: + +- **Not public (no underscore).** A bare name is an implicit stability promise. + The seam is still evolving; we don't want to guarantee it. +- **Not dunder (`__name`).** Double underscore triggers per-class name + mangling, which breaks subclassing — the exact capability this PR exists to + enable. It's the one "more private" option that defeats the purpose. +- **Single underscore is the idiom for "reachable, unsupported, subclass- + friendly."** Python privacy is convention, not enforcement: a downstream lib + *can* call these, and that's intended. The underscore communicates "no + stability guarantee," nothing more. + +Consequence that drives the rest of the plan: because the underscore does not +enforce anything, **the docstrings are the contract.** They must carry the +invariants as if the methods were public, because for the people who reach them +they effectively are. + +--- + +## Work items + +### 1. Make the docstrings the contract (primary work) +Each exposed primitive states its ownership transfer explicitly: +- `_activate(handle)` — takes ownership; frees on close; only from + `UNINITIALIZED`; rejects null and double-activation. +- `_swap_handle(new_handle)` — **the caller guarantees the callee already + consumed and freed the old pointer.** Requires `ACTIVE`; rejects null. +- `_mark_consumed()` — the native pointer is now owned elsewhere; this releases + only *Python-side* resources and marks `CLOSED` without freeing the pointer. +- `_wrap_native_handle(handle)` — ownership transfers **only on successful + return**; if it raises, the caller still owns the pointer and must free it. + +### 2. Nail the `_wrap_native_handle` initialization invariant +It bypasses `__init__` entirely and runs only `_init_attrs()`. Add one explicit +line to the docstring: + +> Everything an instance needs besides the native handle must be set in +> `_init_attrs()`, not `__init__` — `_wrap_native_handle` never runs `__init__`. + +Live proof this matters: `Reader.__init__` sets `self._context = context` +*after* `_init_attrs()`, so a Reader built via `_wrap_native_handle` would get +`_context = None`. Fine for `Builder.from_archive` (no context by design), but a +footgun for any external extender who puts setup in `__init__`. + +### 3. Legibility on the consume-and-return null path +In `with_fragment` and `with_archive`, the null branch relies on +`_check_ffi_operation_result` raising *before* control reaches `_swap_handle`. +It is safe today, but reads as "mark consumed, then a check that happens to +raise." Make the intent explicit: + +```python +new_ptr = _lib.c2pa_..._with_...(self._handle, ...) +if not new_ptr: + # callee consumed the old handle and returned nothing to own + self._mark_consumed() + _check_ffi_operation_result(new_ptr, "...") # raises here +self._swap_handle(new_ptr) +``` + +Moving the check inside the `if not new_ptr:` block makes the control flow say +what it means. No behavior change. + +### 4. `super().__init__()` audit (cheap) +Every `_activate` caller depends on `self._lifecycle_state` already existing +(set by `ManagedResource.__init__`). Context / Reader / Builder / Signer visibly +call `super().__init__()`. Confirm `Settings.__init__` does too. If the existing +tests pass, it already does — this is a five-second eyeball, not a suspected bug. + +### 5. Tests: cover the *external-extender* path +The added lifecycle / integration / cross-thread tests cover the built-in types. +Add the case the PR actually unlocks: +- A minimal subclass that owns a raw handle and reaches the lifecycle via + `_wrap_native_handle` + `_init_attrs`, asserting the instance is fully built + (no missing attributes) and frees exactly once. +- The same wrapped instance under the fork guard: a foreign-process teardown + skips the native free (owner-PID stamped through the wrap path). + +--- + +## Explicit non-goals (pre-empt scope-creep review comments) +- **Not** renaming anything to public. The access level is the decision, not an + oversight. +- **Not** adding fork guards to *operation* paths (`with_fragment` etc.). The + PID guard's contract is teardown-safety in a forked child, not operation- + safety. Operating on a handle in a forked child is caller error and out of + scope. + +--- + +## Review-defense notes (the "why", pre-answered) + +- **Why underscore, not public?** See the extension-model decision above: + reachable but unsupported is exactly the intent. +- **Why does `_mark_consumed()` now run `_release()`?** Previously the consume + paths leaked Python-side resources (callback refs, streams, caches) until GC. + Releasing on consume is a fix, not a regression. It cannot double-release: + `_cleanup_resources` skips when the state is already `CLOSED`. +- **Why is the signer callback copied onto the Context before the signer is + consumed?** The native signer holds a pointer to the Python callback. + `_mark_consumed() -> _release()` clears the signer's callback ref, so the + Context must capture its own reference *first*, or the first invocation of the + consumed signer's callback would be a use-after-free. The ordering is the + whole reason it's safe; the inline comment explaining it must stay. diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 54e3ac38..277082b6 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -292,11 +292,15 @@ def _release(self): """ def _mark_consumed(self): - """Mark as consumed by an FFI call that took ownership - of native resources e.g. pointers. The new owner frees the handle, - but this class's own resources are still ours to let go of, and - marking the resource closed means _cleanup_resources will not do it - later. + """Mark as consumed by an FFI call that took ownership of the native + handle. + + Ownership contract: the native pointer is now owned elsewhere, so this + does not free it. It releases only Python-side resources (via + `_release()`: callback refs, streams, caches) and marks the resource + closed. Marking it closed stops `_cleanup_resources` from freeing the + now foreign-owned pointer later; releasing here means those Python + resources are not stranded until garbage collection. """ if is_foreign_process(self): self._handle = None @@ -347,13 +351,17 @@ def _swap_handle(self, new_handle): """Replace the handle after an FFI call consumed the old one and returned a replacement. - The old pointer is already owned and freed by the callee, - so it is not freed here. + Ownership contract: the caller guarantees the callee has already + consumed and freed the old pointer. This method never frees the old + pointer; it only rebinds `self._handle` to the replacement. Calling it + when the old pointer was not consumed leaks that pointer. A null return from such a call is ambiguous (the callee may have failed validation before taking ownership, or failed the operation after), so callers must not call this with a null replacement. + Requires the resource to be active. + Args: new_handle: Non-null native pointer returned by the FFI call @@ -378,8 +386,13 @@ def _wrap_native_handle(cls, handle): __init__ is bypassed, so `_init_attrs()` supplies the class's own defaults and the instance is stamped with the creating process. - Ownership of `handle` only transfers once this returns. If it raises, - the caller still owns the pointer and must free it. + Everything an instance needs besides the native handle must be set in + `_init_attrs()`, not `__init__` — `_wrap_native_handle` never runs + `__init__`. An attribute a subclass sets only in `__init__` will be + missing (or left at its `_init_attrs` default) on a wrapped instance. + + Ownership of `handle` transfers only on successful return. If this + raises, the caller still owns the pointer and must free it. Args: handle: Non-null native pointer to take ownership of @@ -2565,13 +2578,14 @@ def with_fragment(self, format: str, stream, ) # c2pa_reader_with_fragment consumed the old handle. A null return - # leaves no replacement to take ownership of. + # leaves no replacement to take ownership of: mark consumed, then + # raise. _swap_handle is only reached when new_ptr is non-null. if not new_ptr: self._mark_consumed() - _check_ffi_operation_result(new_ptr, - Reader._ERROR_MESSAGES[ - 'fragment_error' - ].format("Unknown error")) + _check_ffi_operation_result(new_ptr, + Reader._ERROR_MESSAGES[ + 'fragment_error' + ].format("Unknown error")) self._swap_handle(new_ptr) # Invalidate caches: processing a new BMFF fragment updates the native @@ -3543,11 +3557,12 @@ def with_archive(self, stream: Any) -> 'Builder': f"Error loading archive: {e}" ) # c2pa_builder_with_archive consumed the old handle. A null return - # leaves no replacement to take ownership of. + # leaves no replacement to take ownership of: mark consumed, then + # raise. _swap_handle is only reached when new_ptr is non-null. if not new_ptr: self._mark_consumed() - _check_ffi_operation_result( - new_ptr, "Failed to load archive into builder") + _check_ffi_operation_result( + new_ptr, "Failed to load archive into builder") self._swap_handle(new_ptr) return self diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 43bbcb05..fdea7041 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -6960,6 +6960,24 @@ def __init__(self): def _release(self): self.release_calls += 1 + class _ExtenderResource(ManagedResource): + """A downstream extender that owns a raw handle and wraps it via + _wrap_native_handle. It carries several attributes of its own, all + defaulted in _init_attrs (not __init__), and _release reads them, so a + missing attribute would surface as an AttributeError on teardown. + """ + + def _init_attrs(self): + super()._init_attrs() + self.label = "extender" + self.buffer = [] + self.released = False + + def _release(self): + # Reads attributes _init_attrs is responsible for defaulting. + self.buffer.append(self.label) + self.released = True + def setUp(self): self.data_dir = FIXTURES_DIR self.freed = [] @@ -7251,6 +7269,40 @@ def test_mark_consumed_in_foreign_process_skips_release(self): self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) self.assertIsNone(res._handle) + def test_extender_wraps_handle_fully_built(self): + obj = self._ExtenderResource._wrap_native_handle(0xE0) + + # Every attribute _init_attrs defaults is present, + # even thoug __init__ never ran. + self.assertEqual(obj.label, "extender") + self.assertEqual(obj.buffer, []) + self.assertFalse(obj.released) + self.assertTrue(obj.is_valid) + self.assertEqual(obj._owner_pid, os.getpid()) + + # _release reads those attributes, so a missing one would raise here. + obj.close() + obj.close() + + self.assertTrue(obj.released) + self.assertEqual(obj.buffer, ["extender"]) + self.assertEqual(self.freed, [0xE0], "wrapped handle freed once") + + def test_extender_foreign_teardown_skips_native_free(self): + obj = self._ExtenderResource._wrap_native_handle(0xE1) + # Stamp a foreign owner: + # teardown runs in a process that did not create the handle, + # so it must not free the pointer or release. + obj._owner_pid = os.getpid() + 1 + + obj.close() + + self.assertEqual(self.freed, [], + "forked child freed a handle its parent still owns") + self.assertFalse(obj.released, "foreign teardown ran _release") + self.assertEqual(obj._lifecycle_state, LifecycleState.ACTIVE) + self.assertEqual(obj._handle, 0xE1) + def test_signer_init_rejects_null_pointer(self): with self.assertRaises(Error): Signer(None) @@ -7272,8 +7324,8 @@ def test_builder_from_archive_wraps_handle(self): def test_context_build_failure_consumes_signer(self): # c2pa_context_builder_set_signer takes ownership of the signer # pointer immediately, so a later build failure must still leave the - # Signer consumed. Otherwise it holds a pointer the native side has - # already freed. + # Signer consumed. + # Otherwise it holds a pointer the native side has already freed. self._use_real_frees() signer = self._make_signer() real_build = c2pa_module._lib.c2pa_context_builder_build From d6ebd6fade918ca91eedb5297139abe442d704ab Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:47:17 -0700 Subject: [PATCH 11/67] fix: Improve pointers handling 5 --- examples/read.py | 2 +- examples/sign.py | 2 +- examples/sign_info.py | 4 +- src/c2pa/c2pa.py | 63 ++++++++++++++++++----------- tests/perf/scenarios.py | 33 ++++++++++++++++ tests/test_unit_tests.py | 66 +++++++++++++++++++++++++++---- tests/test_unit_tests_threaded.py | 13 ++++-- 7 files changed, 146 insertions(+), 37 deletions(-) diff --git a/examples/read.py b/examples/read.py index e4b718a9..b2ef9fc6 100644 --- a/examples/read.py +++ b/examples/read.py @@ -36,7 +36,7 @@ def read_c2pa_data(media_path: str): # All objects using this context will have trust configured. with c2pa.Context(settings) as context: with c2pa.Reader(media_path, context=context) as reader: - print(reader.detailed_json()) + print(reader.crjson()) except Exception as e: print(f"Error reading C2PA data from {media_path}: {e}") sys.exit(1) diff --git a/examples/sign.py b/examples/sign.py index e6c14859..09133b35 100644 --- a/examples/sign.py +++ b/examples/sign.py @@ -110,6 +110,6 @@ def callback_signer_es256(data: bytes) -> bytes: # The validation state will depend on loaded trust settings. # Without loaded trust settings, # the manifest validation_state will be "Invalid". - print(reader.json()) + print(reader.crjson()) print("\nExample completed successfully!") diff --git a/examples/sign_info.py b/examples/sign_info.py index 6b256647..3735ad0b 100644 --- a/examples/sign_info.py +++ b/examples/sign_info.py @@ -40,7 +40,7 @@ print("\nReading existing C2PA metadata:") with open(fixtures_dir + "C.jpg", "rb") as file: with c2pa.Reader("image/jpeg", file) as reader: - print(reader.json()) + print(reader.crjson()) # Create a signer from certificate and key files with open(fixtures_dir + "es256_certs.pem", "rb") as cert_file: @@ -103,7 +103,7 @@ # The validation state will depend on loaded trust settings. # Without loaded trust settings, # the manifest validation_state will be "Invalid". - print(reader.json()) + print(reader.crjson()) print("\nExample completed successfully!") diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 277082b6..b0a24206 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -291,6 +291,23 @@ def _release(self): The default implementation does nothing. """ + def _safe_release(self): + """Run _release(), logging (never raising) if it fails. + + Shared by _mark_consumed and _cleanup_resources so the try/except/log + wrapper lives in one place. Each caller keeps its own lifecycle-state + transition, because the two paths deliberately order the CLOSED flip + differently relative to this call (see each call site). + """ + try: + self._release() + except Exception: + logger.error( + "Failed to release %s resources", + type(self).__name__, + exc_info=True, + ) + def _mark_consumed(self): """Mark as consumed by an FFI call that took ownership of the native handle. @@ -309,14 +326,7 @@ def _mark_consumed(self): # Callers raise straight after consuming, so a failing _release() # here would mask the error they are reporting. - try: - self._release() - except Exception: - logger.error( - "Failed to release %s resources", - type(self).__name__, - exc_info=True, - ) + self._safe_release() self._handle = None self._lifecycle_state = LifecycleState.CLOSED @@ -411,6 +421,15 @@ def _cleanup_resources(self): """Release native resources idempotently.""" try: if is_foreign_process(self): + # A forked child holds a separate copy of this object and the + # parent still owns the real handle and frees it. Mark this + # copy closed and null its handle so the child cannot mistake + # it for usable or free it, but do not free here. + # Mutating this copy does not touch the parent's. + if hasattr(self, '_handle'): + self._handle = None + if hasattr(self, '_lifecycle_state'): + self._lifecycle_state = LifecycleState.CLOSED return if ( hasattr(self, '_lifecycle_state') @@ -420,14 +439,7 @@ def _cleanup_resources(self): # A failing _release() must not skip the free below: # that would strand the native handle on an object already # marked CLOSED, making it unreachable and unfreeable. - try: - self._release() - except Exception: - logger.error( - "Failed to release %s resources", - type(self).__name__, - exc_info=True, - ) + self._safe_release() if hasattr(self, '_handle') and self._handle: try: ManagedResource._free_native_ptr(self._handle) @@ -2570,12 +2582,19 @@ def with_fragment(self, format: str, stream, ) with Stream(stream) as main_obj, Stream(fragment_stream) as frag_obj: - new_ptr = _lib.c2pa_reader_with_fragment( - self._handle, - format_bytes, - main_obj._stream, - frag_obj._stream, - ) + try: + new_ptr = _lib.c2pa_reader_with_fragment( + self._handle, + format_bytes, + main_obj._stream, + frag_obj._stream, + ) + except Exception as e: + # The callee consumes the old handle before it can fail, so + # treat it as consumed and let go of resources. + self._mark_consumed() + raise C2paError( + Reader._ERROR_MESSAGES['fragment_error'].format(e)) # c2pa_reader_with_fragment consumed the old handle. A null return # leaves no replacement to take ownership of: mark consumed, then diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index b4b7e46c..25c688a0 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -957,6 +957,37 @@ def scenario_fork_parent_frees_after_fork(iterations: int = 100) -> None: r.close() +def scenario_fork_child_closes_then_parent_frees(iterations: int = 100) -> None: + """Fork safety benchmark scenario: + 20 Readers created, fork, the CHILD closes all 20 inherited Readers (and + runs GC so any __del__ fires) before exiting, then the parent closes its + own 20 copies. Exercises the child-side path where _cleanup_resources marks + the child's copy CLOSED and nulls the handle while skipping the native free + — the branch scenario_fork_parent_frees_after_fork never hits (its child + does nothing). Two invariants: the child must exit cleanly (no deadlock via + the 5 s alarm, no crash from a child-side double-free), and the parent must + still free all 20 (leaked_bytes stays at baseline — the child's state + mutation does not suppress the parent's frees, since the copies are + independent post-fork). + """ + if not hasattr(os, "fork"): + return + for _ in _iterate(iterations): + readers = [] + for _ in range(20): + with open(SIGNED_JPEG, "rb") as f: + readers.append(Reader("image/jpeg", f)) + + def _child(): + for r in readers: + r.close() # foreign teardown: mark closed, skip native free + gc.collect() + + _fork_wait(_child) + for r in readers: + r.close() # parent's own copies: real free + + def scenario_fork_child_sys_exit(iterations: int = 100) -> None: """Fork safety benchmark scenario: Child calls sys.exit(0), full Python shutdown: atexit, finalizers, GC. @@ -1191,6 +1222,8 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "fork_thread_local_orphan": scenario_fork_thread_local_orphan, "fork_gc_cycle": scenario_fork_gc_cycle, "fork_parent_frees_after_fork": scenario_fork_parent_frees_after_fork, + "fork_child_closes_then_parent_frees": + scenario_fork_child_closes_then_parent_frees, "fork_child_sys_exit": scenario_fork_child_sys_exit, "fork_stream_cleanup": scenario_fork_stream_cleanup, "fork_swap_cleanup": scenario_fork_swap_cleanup, diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index fdea7041..466d5a81 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -7189,10 +7189,19 @@ def test_foreign_child_skips_free_for_wrapped_and_swapped(self): self.assertEqual(self.freed, [], "forked child freed a pointer its parent still owns") - # The parent still owns these handles, - # so the child must not mark the objects dead either. - self.assertEqual(wrapped._lifecycle_state, LifecycleState.ACTIVE) - self.assertEqual(swapped._handle, 0xC3) + # The child must not free a pointer the parent still owns. + # The child does mark its own copies closed and nulls their handles, + # which is safe (the parent holds a separate copy) and + # stops the child from reusing a parent-owned handle. + self.assertEqual(wrapped._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(wrapped._handle) + self.assertEqual(swapped._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(swapped._handle) + + # A second foreign teardown is a no-op: still nothing freed. + wrapped.close() + swapped.close() + self.assertEqual(self.freed, []) def test_owning_process_frees_wrapped_and_swapped_exactly_once(self): wrapped = self._FakeHandleResource._wrap_native_handle(0xC4) @@ -7292,7 +7301,8 @@ def test_extender_foreign_teardown_skips_native_free(self): obj = self._ExtenderResource._wrap_native_handle(0xE1) # Stamp a foreign owner: # teardown runs in a process that did not create the handle, - # so it must not free the pointer or release. + # so it must not free the pointer or run _release (which could touch + # native streams and deadlock after a multithreaded fork). obj._owner_pid = os.getpid() + 1 obj.close() @@ -7300,8 +7310,18 @@ def test_extender_foreign_teardown_skips_native_free(self): self.assertEqual(self.freed, [], "forked child freed a handle its parent still owns") self.assertFalse(obj.released, "foreign teardown ran _release") - self.assertEqual(obj._lifecycle_state, LifecycleState.ACTIVE) - self.assertEqual(obj._handle, 0xE1) + # The child marks its own copy closed and nulls the handle: safe (the + # parent holds a separate copy) and it stops the child reusing a + # parent-owned handle. + self.assertEqual(obj._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(obj._handle) + + # A second foreign teardown stays a no-op, and any operation on the + # now-closed child copy fails loudly instead of reaching native code. + obj.close() + self.assertEqual(self.freed, []) + with self.assertRaises(Error): + obj._ensure_valid_state() def test_signer_init_rejects_null_pointer(self): with self.assertRaises(Error): @@ -7687,6 +7707,38 @@ def test_reader_with_fragment_null_return_consumes_self(self): self.assertEqual(self._free_count(freed, consumed_handle), 0, "close() freed a handle the FFI already consumed") + def test_reader_with_fragment_ffi_raise_consumes_self(self): + # If the ctypes call itself raises (not a null return), the callee has + # already consumed the old handle, so with_fragment must mark self + # consumed rather than leave a dangling pointer that close() would + # double-free. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + consumed_handle = reader._handle + + def _raise(*_args): + raise RuntimeError("boom") + + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = _raise + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("video/mp4", init, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + + self.assertIsNone(reader._handle) + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + + freed = self._instrument_frees() + reader.close() + self.assertEqual(self._free_count(freed, consumed_handle), 0, + "close() freed a handle the FFI already consumed") + # Backfilling a pointer minted by a direct FFI call. Builder.from_archive # is the only production caller of _wrap_native_handle, so these are the # only tests that drive the primitive as the generic entry point it is. diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 44609adc..540dfc82 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -96,12 +96,17 @@ def test_no_stamp_calls_free(self): obj._cleanup_resources() mock_free.assert_called_once() - def test_foreign_pid_leaves_state_unchanged(self): - """Guard returns early; lifecycle state stays ACTIVE (not CLOSED).""" + def test_foreign_pid_marks_closed_without_free(self): + """A foreign child skips the native free but marks its own copy closed + and nulls the handle, so the child cannot reuse a parent-owned handle. + The parent holds a separate copy and frees it independently. + """ obj = _make_resource(pid_offset=1) - with patch('c2pa.c2pa._lib'): + with patch.object(ManagedResource, '_free_native_ptr') as mock_free: obj._cleanup_resources() - self.assertEqual(obj._lifecycle_state, LifecycleState.ACTIVE) + mock_free.assert_not_called() + self.assertEqual(obj._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(obj._handle) def test_double_cleanup_is_idempotent(self): """Second call is a no-op after successful first cleanup.""" From fad717bf80502312c6017b9843ab8a4bd390aeb3 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:23:59 -0700 Subject: [PATCH 12/67] fix: Rebaseline --- src/c2pa/c2pa.py | 2 +- tests/perf/baseline.json | 345 ++++++++++++++++++++------------------- 2 files changed, 176 insertions(+), 171 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index b0a24206..fb7f4cf5 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -2591,7 +2591,7 @@ def with_fragment(self, format: str, stream, ) except Exception as e: # The callee consumes the old handle before it can fail, so - # treat it as consumed and let go of resources. + # treated as consumed to let go of resources. self._mark_consumed() raise C2paError( Reader._ERROR_MESSAGES['fragment_error'].format(e)) diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index 934a6390..a6a75177 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -2,269 +2,274 @@ "_meta": { "memray_version": "1.19.3", "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.89.0", - "iterations": 100, + "c2pa_native_version": "c2pa-v0.90.0", + "iterations": 200, "perf_env": "python-3.12-slim", "arch": "aarch64" }, "reader_jpeg_legacy": { - "peak_bytes": 3791301, - "leaked_bytes": 3292672, - "total_allocations": 722823 + "peak_bytes": 3830666, + "leaked_bytes": 3331494, + "total_allocations": 1365464 }, "reader_jpeg_with_context": { - "peak_bytes": 3785583, - "leaked_bytes": 3284873, - "total_allocations": 717272 + "peak_bytes": 3824947, + "leaked_bytes": 3324219, + "total_allocations": 1354423 }, "reader_manifest_data_context": { - "peak_bytes": 7576945, - "leaked_bytes": 3407648, - "total_allocations": 619356 + "peak_bytes": 7616236, + "leaked_bytes": 3448019, + "total_allocations": 1152354 }, "reader_mp4": { - "peak_bytes": 4159582, - "leaked_bytes": 3283805, - "total_allocations": 2089508 + "peak_bytes": 4200644, + "leaked_bytes": 3323805, + "total_allocations": 4100459 }, "reader_wav": { - "peak_bytes": 4464615, - "leaked_bytes": 3293702, - "total_allocations": 413484 + "peak_bytes": 4501138, + "leaked_bytes": 3333747, + "total_allocations": 746935 }, "builder_sign_jpeg_legacy": { - "peak_bytes": 7724599, - "leaked_bytes": 3408513, - "total_allocations": 560691 + "peak_bytes": 8108426, + "leaked_bytes": 3485229, + "total_allocations": 1115105 }, "builder_sign_jpeg_with_context": { - "peak_bytes": 7716613, - "leaked_bytes": 3400514, - "total_allocations": 554788 + "peak_bytes": 8108408, + "leaked_bytes": 3477526, + "total_allocations": 1103189 }, "builder_sign_png_legacy": { - "peak_bytes": 7961139, - "leaked_bytes": 3406984, - "total_allocations": 1981843 + "peak_bytes": 8002675, + "leaked_bytes": 3447638, + "total_allocations": 3887535 }, "builder_sign_png_with_context": { - "peak_bytes": 7955799, - "leaked_bytes": 3401929, - "total_allocations": 1975867 + "peak_bytes": 7994768, + "leaked_bytes": 3440151, + "total_allocations": 3875483 }, "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 44002804, - "leaked_bytes": 3801899, - "total_allocations": 566497 + "peak_bytes": 44042681, + "leaked_bytes": 3841702, + "total_allocations": 1045285 }, "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 45784452, - "leaked_bytes": 3800550, - "total_allocations": 565361 + "peak_bytes": 45824424, + "leaked_bytes": 3840908, + "total_allocations": 1043917 }, "builder_sign_png_parallel_split_pool": { - "peak_bytes": 46054163, - "leaked_bytes": 3801867, - "total_allocations": 1987868 + "peak_bytes": 46094336, + "leaked_bytes": 3860177, + "total_allocations": 3887276 }, "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 44206900, - "leaked_bytes": 3800490, - "total_allocations": 1986526 + "peak_bytes": 46062355, + "leaked_bytes": 3876678, + "total_allocations": 3885976 }, "builder_sign_gif": { - "peak_bytes": 14574556, - "leaked_bytes": 3400735, - "total_allocations": 8550061 + "peak_bytes": 14614794, + "leaked_bytes": 3439807, + "total_allocations": 17023655 }, "builder_sign_heic": { - "peak_bytes": 4644811, - "leaked_bytes": 3407275, - "total_allocations": 831824 + "peak_bytes": 4677761, + "leaked_bytes": 3447623, + "total_allocations": 1569619 }, "builder_sign_m4a": { - "peak_bytes": 18885052, - "leaked_bytes": 3407378, - "total_allocations": 2647270 + "peak_bytes": 18812724, + "leaked_bytes": 3448162, + "total_allocations": 5200482 }, "builder_sign_webp": { - "peak_bytes": 8928848, - "leaked_bytes": 3399564, - "total_allocations": 499272 + "peak_bytes": 8970587, + "leaked_bytes": 3440333, + "total_allocations": 922037 }, "builder_sign_avi": { - "peak_bytes": 7068483, - "leaked_bytes": 3399340, - "total_allocations": 45032279 + "peak_bytes": 7110163, + "leaked_bytes": 3440347, + "total_allocations": 89988101 }, "builder_sign_mp4": { - "peak_bytes": 6198764, - "leaked_bytes": 3407319, - "total_allocations": 1944326 + "peak_bytes": 6224729, + "leaked_bytes": 3448147, + "total_allocations": 3794640 }, "builder_sign_tiff": { - "peak_bytes": 13151779, - "leaked_bytes": 3400355, - "total_allocations": 5472611 + "peak_bytes": 13192399, + "leaked_bytes": 3440348, + "total_allocations": 10868711 }, "builder_sign_jpeg_parent_of": { - "peak_bytes": 14204315, - "leaked_bytes": 3401481, - "total_allocations": 1292637 + "peak_bytes": 14244190, + "leaked_bytes": 3439412, + "total_allocations": 2514770 }, "builder_sign_jpeg_component_of": { - "peak_bytes": 14204923, - "leaked_bytes": 3400730, - "total_allocations": 1315125 + "peak_bytes": 14246389, + "leaked_bytes": 3440789, + "total_allocations": 2559666 }, "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14588185, - "leaked_bytes": 3549911, - "total_allocations": 2299453 + "peak_bytes": 14597099, + "leaked_bytes": 3546490, + "total_allocations": 4534820 }, "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14506586, - "leaked_bytes": 3401364, - "total_allocations": 2796028 + "peak_bytes": 14545928, + "leaked_bytes": 3440216, + "total_allocations": 5528067 }, "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14601941, - "leaked_bytes": 3558116, - "total_allocations": 2289245 + "peak_bytes": 14539155, + "leaked_bytes": 3544510, + "total_allocations": 4508197 }, "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14503695, - "leaked_bytes": 3401276, - "total_allocations": 2785702 + "peak_bytes": 14544717, + "leaked_bytes": 3441158, + "total_allocations": 5501353 }, "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14236247, - "leaked_bytes": 3420354, - "total_allocations": 1777705 + "peak_bytes": 14277459, + "leaked_bytes": 3460332, + "total_allocations": 3480521 + }, + "builder_from_archive_roundtrip": { + "peak_bytes": 14277245, + "leaked_bytes": 3460181, + "total_allocations": 3108863 + }, + "builder_with_archive_swap": { + "peak_bytes": 3660981, + "leaked_bytes": 3330239, + "total_allocations": 708348 + }, + "reader_with_fragment_swap": { + "peak_bytes": 3758666, + "leaked_bytes": 3332314, + "total_allocations": 3792727 }, "builder_to_archive_with_ingredient": { - "peak_bytes": 14010137, - "leaked_bytes": 3278101, - "total_allocations": 957452 + "peak_bytes": 14049708, + "leaked_bytes": 3318104, + "total_allocations": 1836413 }, "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": { - "peak_bytes": 14225579, - "leaked_bytes": 3420822, - "total_allocations": 2981495 + "peak_bytes": 14264719, + "leaked_bytes": 3459571, + "total_allocations": 5893306 }, "builder_write_ingredient_archive": { - "peak_bytes": 14010131, - "leaked_bytes": 3278099, - "total_allocations": 944654 + "peak_bytes": 14049702, + "leaked_bytes": 3318102, + "total_allocations": 1810851 }, "builder_sign_jpeg_add_ingredient_from_archive": { - "peak_bytes": 14075511, - "leaked_bytes": 3421125, - "total_allocations": 1753642 + "peak_bytes": 14113307, + "leaked_bytes": 3459596, + "total_allocations": 3424465 }, "builder_ingredient_archive_roundtrip": { - "peak_bytes": 14222976, - "leaked_bytes": 3419885, - "total_allocations": 2609017 + "peak_bytes": 14264391, + "leaked_bytes": 3460351, + "total_allocations": 5146608 }, "builder_sign_jpeg_two_ingredient_archives": { - "peak_bytes": 14075190, - "leaked_bytes": 3421103, - "total_allocations": 2161232 + "peak_bytes": 14114874, + "leaked_bytes": 3461104, + "total_allocations": 4226879 }, "reader_error_no_manifest": { - "peak_bytes": 3504260, - "leaked_bytes": 3263283, - "total_allocations": 178873 + "peak_bytes": 3544227, + "leaked_bytes": 3302551, + "total_allocations": 278514 }, "builder_error_invalid_manifest": { - "peak_bytes": 3292723, - "leaked_bytes": 3235293, - "total_allocations": 96622 + "peak_bytes": 3331458, + "leaked_bytes": 3276705, + "total_allocations": 114863 }, "reader_string_apis": { - "peak_bytes": 3915891, - "leaked_bytes": 3284213, - "total_allocations": 1185492 + "peak_bytes": 3957555, + "leaked_bytes": 3324853, + "total_allocations": 2296696 + }, + "signer_construction": { + "peak_bytes": 3331349, + "leaked_bytes": 3268750, + "total_allocations": 155984 }, "fork_reader_collect": { - "peak_bytes": 3789318, - "leaked_bytes": 3291267, - "total_allocations": 705123 + "peak_bytes": 3830817, + "leaked_bytes": 3333241, + "total_allocations": 1330058 }, "fork_contended_mutex": { - "peak_bytes": 7617864, - "leaked_bytes": 3421711, - "total_allocations": 33591148 + "peak_bytes": 7659277, + "leaked_bytes": 3461442, + "total_allocations": 67635759 }, "fork_thread_local_orphan": { - "peak_bytes": 3876269, - "leaked_bytes": 3379616, - "total_allocations": 731511 + "peak_bytes": 3916484, + "leaked_bytes": 3419968, + "total_allocations": 1383197 }, "fork_gc_cycle": { - "peak_bytes": 3790340, - "leaked_bytes": 3291400, - "total_allocations": 706802 + "peak_bytes": 3830375, + "leaked_bytes": 3332761, + "total_allocations": 1334044 }, "fork_parent_frees_after_fork": { - "peak_bytes": 5989167, - "leaked_bytes": 3289951, - "total_allocations": 12461253 + "peak_bytes": 5427183, + "leaked_bytes": 3329834, + "total_allocations": 24873000 + }, + "fork_child_closes_then_parent_frees": { + "peak_bytes": 5427152, + "leaked_bytes": 3329841, + "total_allocations": 24872992 }, "fork_child_sys_exit": { - "peak_bytes": 3789334, - "leaked_bytes": 3291456, - "total_allocations": 708625 + "peak_bytes": 3831271, + "leaked_bytes": 3332312, + "total_allocations": 1338865 }, "fork_stream_cleanup": { - "peak_bytes": 3402893, - "leaked_bytes": 3230663, - "total_allocations": 93141 - }, - "builder_from_archive_roundtrip": { - "peak_bytes": 14258579, - "leaked_bytes": 3436798, - "total_allocations": 1593617 - }, - "signer_construction": { - "peak_bytes": 3307608, - "leaked_bytes": 3243306, - "total_allocations": 117601 - }, - "builder_with_archive_swap": { - "peak_bytes": 3635924, - "leaked_bytes": 3305070, - "total_allocations": 395447 - }, - "reader_with_fragment_swap": { - "peak_bytes": 3733349, - "leaked_bytes": 3306787, - "total_allocations": 1936244 + "peak_bytes": 3444268, + "leaked_bytes": 3272486, + "total_allocations": 106279 }, "fork_swap_cleanup": { - "peak_bytes": 3639043, - "leaked_bytes": 3308397, - "total_allocations": 401032 - }, - "swap_chain_churn": { - "peak_bytes": 3650097, - "leaked_bytes": 3319300, - "total_allocations": 379064 - }, - "fork_consumed_signer": { - "peak_bytes": 3321464, - "leaked_bytes": 3258207, - "total_allocations": 130030 + "peak_bytes": 3661376, + "leaked_bytes": 3330987, + "total_allocations": 718355 }, "fork_contended_mutex_swap": { - "peak_bytes": 7281970, - "leaked_bytes": 3443799, - "total_allocations": 18799685 + "peak_bytes": 7276901, + "leaked_bytes": 3443869, + "total_allocations": 37514894 }, "fork_contended_mutex_wrap": { - "peak_bytes": 7268578, - "leaked_bytes": 3442991, - "total_allocations": 17791158 + "peak_bytes": 7262615, + "leaked_bytes": 3474333, + "total_allocations": 35888503 + }, + "fork_consumed_signer": { + "peak_bytes": 3331615, + "leaked_bytes": 3269742, + "total_allocations": 179994 + }, + "swap_chain_churn": { + "peak_bytes": 3661193, + "leaked_bytes": 3330367, + "total_allocations": 673717 } } \ No newline at end of file From 075fb21c75920bb447efb39e13fdd31cd31d1979 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 08:15:59 -0700 Subject: [PATCH 13/67] fix: Clean up debug --- pr294-finishing-plan.md | 121 ---------------------------------------- 1 file changed, 121 deletions(-) delete mode 100644 pr294-finishing-plan.md diff --git a/pr294-finishing-plan.md b/pr294-finishing-plan.md deleted file mode 100644 index 3579c47c..00000000 --- a/pr294-finishing-plan.md +++ /dev/null @@ -1,121 +0,0 @@ -# PR #294 — Finishing plan: native-handle lifecycle extension surface - -**Goal:** take #294 from WIP to mergeable and reviewer-proof. The code is -mechanically sound; the work left is *committing to the extension contract* and -writing it down where it can be enforced. No functional rewrite required. - ---- - -## Decision to record first: the extension model - -Downstream libraries should be able to wrap their own FFI pointers in the -managed lifecycle, **without** the project promising a stable public API. - -Chosen: **single-underscore, subclass-facing (protected).** Keep the names as -they are (`_activate`, `_swap_handle`, `_mark_consumed`, `_wrap_native_handle`, -`_init_attrs`). - -Why this level and not the alternatives — state this in the PR so it isn't -re-litigated in review: - -- **Not public (no underscore).** A bare name is an implicit stability promise. - The seam is still evolving; we don't want to guarantee it. -- **Not dunder (`__name`).** Double underscore triggers per-class name - mangling, which breaks subclassing — the exact capability this PR exists to - enable. It's the one "more private" option that defeats the purpose. -- **Single underscore is the idiom for "reachable, unsupported, subclass- - friendly."** Python privacy is convention, not enforcement: a downstream lib - *can* call these, and that's intended. The underscore communicates "no - stability guarantee," nothing more. - -Consequence that drives the rest of the plan: because the underscore does not -enforce anything, **the docstrings are the contract.** They must carry the -invariants as if the methods were public, because for the people who reach them -they effectively are. - ---- - -## Work items - -### 1. Make the docstrings the contract (primary work) -Each exposed primitive states its ownership transfer explicitly: -- `_activate(handle)` — takes ownership; frees on close; only from - `UNINITIALIZED`; rejects null and double-activation. -- `_swap_handle(new_handle)` — **the caller guarantees the callee already - consumed and freed the old pointer.** Requires `ACTIVE`; rejects null. -- `_mark_consumed()` — the native pointer is now owned elsewhere; this releases - only *Python-side* resources and marks `CLOSED` without freeing the pointer. -- `_wrap_native_handle(handle)` — ownership transfers **only on successful - return**; if it raises, the caller still owns the pointer and must free it. - -### 2. Nail the `_wrap_native_handle` initialization invariant -It bypasses `__init__` entirely and runs only `_init_attrs()`. Add one explicit -line to the docstring: - -> Everything an instance needs besides the native handle must be set in -> `_init_attrs()`, not `__init__` — `_wrap_native_handle` never runs `__init__`. - -Live proof this matters: `Reader.__init__` sets `self._context = context` -*after* `_init_attrs()`, so a Reader built via `_wrap_native_handle` would get -`_context = None`. Fine for `Builder.from_archive` (no context by design), but a -footgun for any external extender who puts setup in `__init__`. - -### 3. Legibility on the consume-and-return null path -In `with_fragment` and `with_archive`, the null branch relies on -`_check_ffi_operation_result` raising *before* control reaches `_swap_handle`. -It is safe today, but reads as "mark consumed, then a check that happens to -raise." Make the intent explicit: - -```python -new_ptr = _lib.c2pa_..._with_...(self._handle, ...) -if not new_ptr: - # callee consumed the old handle and returned nothing to own - self._mark_consumed() - _check_ffi_operation_result(new_ptr, "...") # raises here -self._swap_handle(new_ptr) -``` - -Moving the check inside the `if not new_ptr:` block makes the control flow say -what it means. No behavior change. - -### 4. `super().__init__()` audit (cheap) -Every `_activate` caller depends on `self._lifecycle_state` already existing -(set by `ManagedResource.__init__`). Context / Reader / Builder / Signer visibly -call `super().__init__()`. Confirm `Settings.__init__` does too. If the existing -tests pass, it already does — this is a five-second eyeball, not a suspected bug. - -### 5. Tests: cover the *external-extender* path -The added lifecycle / integration / cross-thread tests cover the built-in types. -Add the case the PR actually unlocks: -- A minimal subclass that owns a raw handle and reaches the lifecycle via - `_wrap_native_handle` + `_init_attrs`, asserting the instance is fully built - (no missing attributes) and frees exactly once. -- The same wrapped instance under the fork guard: a foreign-process teardown - skips the native free (owner-PID stamped through the wrap path). - ---- - -## Explicit non-goals (pre-empt scope-creep review comments) -- **Not** renaming anything to public. The access level is the decision, not an - oversight. -- **Not** adding fork guards to *operation* paths (`with_fragment` etc.). The - PID guard's contract is teardown-safety in a forked child, not operation- - safety. Operating on a handle in a forked child is caller error and out of - scope. - ---- - -## Review-defense notes (the "why", pre-answered) - -- **Why underscore, not public?** See the extension-model decision above: - reachable but unsupported is exactly the intent. -- **Why does `_mark_consumed()` now run `_release()`?** Previously the consume - paths leaked Python-side resources (callback refs, streams, caches) until GC. - Releasing on consume is a fix, not a regression. It cannot double-release: - `_cleanup_resources` skips when the state is already `CLOSED`. -- **Why is the signer callback copied onto the Context before the signer is - consumed?** The native signer holds a pointer to the Python callback. - `_mark_consumed() -> _release()` clears the signer's callback ref, so the - Context must capture its own reference *first*, or the first invocation of the - consumed signer's callback would be a use-after-free. The ordering is the - whole reason it's safe; the inline comment explaining it must stay. From 2a78a766d6cf2d41d11a7d438fddcd60b1f440f6 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:17:11 -0700 Subject: [PATCH 14/67] fix: Refactor --- src/c2pa/c2pa.py | 305 ++++++++++-------- tests/perf/scenarios.py | 212 +++++++++++++ tests/test_unit_tests.py | 652 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 1037 insertions(+), 132 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index fb7f4cf5..ee06d265 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -259,7 +259,6 @@ def _init_attrs(self): def __init__(self): self._lifecycle_state = LifecycleState.UNINITIALIZED self._handle = None - _clear_error_state() record_owner_pid(self) @staticmethod @@ -282,7 +281,6 @@ def _ensure_valid_state(self): if not self._handle: raise C2paError( f"{name} has an invalid internal state (active but no handle)") - _clear_error_state() def _release(self): """Override to free class-specific resources (streams, caches, etc.). @@ -293,11 +291,6 @@ def _release(self): def _safe_release(self): """Run _release(), logging (never raising) if it fails. - - Shared by _mark_consumed and _cleanup_resources so the try/except/log - wrapper lives in one place. Each caller keeps its own lifecycle-state - transition, because the two paths deliberately order the CLOSED flip - differently relative to this call (see each call site). """ try: self._release() @@ -388,6 +381,70 @@ def _swap_handle(self, new_handle): self._handle = new_handle + # Errors set by native lib, hinting at the cause of the error + # These errors here means the pointer got somehow rejected by the lib, + # so it is still ours to deal with. + _PRE_CONSUME_ERROR_TAGS = ("UntrackedPointer:", "WrongPointerType:") + + def _consume_and_swap(self, ffi_call, error_message): + """Run an FFI call that consumes this handle and returns a replacement. + + The native lib takes ownership partway through the call, + so a null return can be ambiguous. + The native error tells the cases apart: + - a pointer rejection (`_PRE_CONSUME_ERROR_TAGS`) precedes the + transfer, so the handle is still ours and is kept; + - any other error means it was taken, then the operation failed; + - a null with no error cannot be placed, so the handle is dropped + anyway: leaking is recoverable, double-freeing is not. No native + path does this, so it only appears under mocks. + + The error is read without clearing it first. + The slot is sticky: c2pa_error() peeks and nothing empties it. + + Args: + ffi_call: Callable taking the current handle, returning the + replacement or null. + error_message: Format string with one placeholder, used when the + native layer offers no error of its own. + + Raises: + C2paError: If the call fails, typed by native error when one. + """ + try: + new_ptr = ffi_call(self._handle) + except ctypes.ArgumentError: + # Marshalling failed, so the call never reached the native side + # and the handle is untouched. + raise + except BaseException as e: + self._mark_consumed() + raise C2paError(error_message.format(e)) from e + + if new_ptr: + self._swap_handle(new_ptr) + return + + error = _read_native_error() + if error: + if any(tag in error + for tag in ManagedResource._PRE_CONSUME_ERROR_TAGS): + # Rejected before ownership transferred, so the handle is + # still ours: keep it and let normal cleanup free it. + logger.warning( + "%s: native call rejected the handle before taking " + "ownership (%s); handle retained", + type(self).__name__, + error) + _raise_typed_c2pa_error(error) + + # Ownership transferred and then the operation failed. + self._mark_consumed() + _raise_typed_c2pa_error(error) + + self._mark_consumed() + raise C2paError(error_message.format("Unknown error")) + @classmethod def _wrap_native_handle(cls, handle): """Build a brand-new instance around an already-valid, @@ -566,18 +623,25 @@ class C2paStream(ctypes.Structure): ] -def _clear_error_state(): - """Clear any existing error state from the C library. +def _read_native_error() -> Optional[str]: + """Read the last error from the native library, or None if unset. - This function should be called at the beginning of object initialization - and before any operations that could potentially raise an error, - to ensure that stale error states from previous operations don't interfere - with new objects being created, or independent function calls. + Peeks: the error stays in the native slot, + until the next error overwrites it. + + With no error set the native side still returns an owned pointer to an + empty string, so the pointer alone does not tell us whether there is an + error. Only a non-empty message counts as one; the empty string still + has to be freed. """ error = _lib.c2pa_error() - if error: - # Free the error to clear the state + if not error: + return None + try: + message = ctypes.string_at(error).decode('utf-8') + finally: _lib.c2pa_string_free(error) + return message or None class C2paSignerInfo(ctypes.Structure): @@ -600,7 +664,6 @@ def __init__(self, alg, sign_cert, private_key, ta_url): private_key: The private key as a string ta_url: The timestamp authority URL as bytes """ - _clear_error_state() if sign_cert is None: raise ValueError("sign_cert must be set") @@ -1010,6 +1073,24 @@ class _C2paVerify(C2paError): pass +class _C2paUntrackedPointer(C2paError): + """Exception raised when the native layer does not recognize a pointer. + + Raised when a consume-and-return call rejects the handle it was given + before taking ownership of it, so the caller still owns that handle. + """ + pass + + +class _C2paWrongPointerType(C2paError): + """Exception raised when a pointer is tracked under a different type. + + Like _C2paUntrackedPointer, this is rejected before ownership transfer, + so the caller still owns the handle it passed in. + """ + pass + + # Attach exception subclasses to C2paError for backward compatibility # Preserves behavior for exception catching like except C2paError.ManifestNotFound, # also reduces imports (think of it as an alias of sorts) @@ -1028,6 +1109,8 @@ class _C2paVerify(C2paError): C2paError.ResourceNotFound = _C2paResourceNotFound C2paError.Signature = _C2paSignature C2paError.Verify = _C2paVerify +C2paError.UntrackedPointer = _C2paUntrackedPointer +C2paError.WrongPointerType = _C2paWrongPointerType class _StringContainer: @@ -1152,6 +1235,10 @@ def _raise_typed_c2pa_error(error_str: str) -> None: raise C2paError.Signature(error_str) elif error_type == "Verify": raise C2paError.Verify(error_str) + elif error_type == "UntrackedPointer": + raise C2paError.UntrackedPointer(error_str) + elif error_type == "WrongPointerType": + raise C2paError.WrongPointerType(error_str) # If no recognized error type, raise base C2paError raise C2paError(error_str) @@ -1179,10 +1266,8 @@ def _parse_operation_result_for_error( """ if not result: # pragma: no cover if check_error: - error = _lib.c2pa_error() - if error: - error_str = ctypes.string_at(error).decode('utf-8') - _lib.c2pa_string_free(error) + error_str = _read_native_error() + if error_str: _raise_typed_c2pa_error(error_str) return None @@ -1314,7 +1399,6 @@ def load_settings(settings: Union[str, dict], format: str = "json") -> None: DeprecationWarning, stacklevel=2, ) - _clear_error_state() # Convert to JSON string as necessary try: @@ -1416,7 +1500,7 @@ def __init__(self): ptr = _lib.c2pa_settings_new() try: _check_ffi_operation_result(ptr, "Failed to create Settings") - except Exception: + except BaseException: if ptr: ManagedResource._free_native_ptr(ptr) raise @@ -1608,15 +1692,18 @@ def __init__( signer._ensure_valid_state() # c2pa_context_builder_set_signer takes ownership of the # signer pointer immediately (Box::from_raw), on its error - # path as well as on success. The Signer is therefore - # consumed below on any result; leaving it owning a freed - # pointer would make any later use of it a use-after-free. + # path as well as on success. self._signer_callback_cb = signer._callback_cb - result = ( - _lib.c2pa_context_builder_set_signer( - builder_ptr, signer._handle, + try: + result = ( + _lib.c2pa_context_builder_set_signer( + builder_ptr, signer._handle, + ) ) - ) + except ctypes.ArgumentError: + # Marshalling failed, so the call never reached the + # native side and the signer was never taken. + raise signer._mark_consumed() if result != 0: _parse_operation_result_for_error(None) @@ -1633,7 +1720,7 @@ def __init__( ) self._activate(ptr) - except Exception: + except BaseException: # Free builder if build was not reached if builder_ptr is not None: try: @@ -1935,7 +2022,6 @@ def flush_callback(ctx): self._flush_cb = FlushCallback(flush_callback) # Create the stream - _clear_error_state() self._stream = _lib.c2pa_create_stream( None, self._read_cb, @@ -1944,8 +2030,9 @@ def flush_callback(ctx): self._flush_cb ) if not self._stream: - error = _parse_operation_result_for_error(_lib.c2pa_error()) - raise Exception("Failed to create stream: {}".format(error)) + error = _read_native_error() + raise C2paError( + "Failed to create stream: {}".format(error or "Unknown error")) self._initialized = True record_owner_pid(self) @@ -2078,15 +2165,14 @@ def _get_supported_mime_types(ffi_func, cache): if cache is not None: return list(cache), cache - _clear_error_state() count = ctypes.c_size_t() arr = ffi_func(ctypes.byref(count)) if not arr: - error = _parse_operation_result_for_error(_lib.c2pa_error()) - if error: - raise C2paError(f"Failed to get supported MIME types: {error}") - return [], cache + error = _read_native_error() + raise C2paError( + "Failed to get supported MIME types: " + f"{error or 'Unknown error'}") if count.value <= 0: try: @@ -2442,11 +2528,16 @@ def _init_from_context(self, context, format_or_path, 'reader_error' ].format("Unknown error") ) - except Exception: + except BaseException: if reader_ptr: ManagedResource._free_native_ptr(reader_ptr) raise + # Adopt the handle before the consuming call: _consume_and_swap + # needs an ACTIVE resource, and from here on normal cleanup owns + # the pointer whichever way the call goes. + self._activate(reader_ptr) + if manifest_data is not None: manifest_array = ( ctypes.c_ubyte * @@ -2454,34 +2545,26 @@ def _init_from_context(self, context, format_or_path, # Consume current reader, # with manifest data and stream (C FFI pattern), # to create a new one (switch out) - new_ptr = ( - _lib.c2pa_reader_with_manifest_data_and_stream( - reader_ptr, - format_bytes, - self._own_stream._stream, - manifest_array, - len(manifest_data), - ) - ) + self._consume_and_swap( + lambda handle: ( + _lib.c2pa_reader_with_manifest_data_and_stream( + handle, + format_bytes, + self._own_stream._stream, + manifest_array, + len(manifest_data), + ) + ), + Reader._ERROR_MESSAGES['reader_error']) else: # Consume reader with stream - new_ptr = _lib.c2pa_reader_with_stream( - reader_ptr, format_bytes, - self._own_stream._stream, - ) - - # reader_ptr has been consumed by the FFI call (freed by it even - # on failure), so there is nothing to free on the error path. - reader_ptr = None - - _check_ffi_operation_result(new_ptr, - Reader._ERROR_MESSAGES[ - 'reader_error' - ].format("Unknown error") - ) - - self._activate(new_ptr) - except Exception: + self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_stream( + handle, format_bytes, + self._own_stream._stream, + ), + Reader._ERROR_MESSAGES['reader_error']) + except BaseException: self._close_streams() raise @@ -2582,30 +2665,14 @@ def with_fragment(self, format: str, stream, ) with Stream(stream) as main_obj, Stream(fragment_stream) as frag_obj: - try: - new_ptr = _lib.c2pa_reader_with_fragment( - self._handle, + self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_fragment( + handle, format_bytes, main_obj._stream, frag_obj._stream, - ) - except Exception as e: - # The callee consumes the old handle before it can fail, so - # treated as consumed to let go of resources. - self._mark_consumed() - raise C2paError( - Reader._ERROR_MESSAGES['fragment_error'].format(e)) - - # c2pa_reader_with_fragment consumed the old handle. A null return - # leaves no replacement to take ownership of: mark consumed, then - # raise. _swap_handle is only reached when new_ptr is non-null. - if not new_ptr: - self._mark_consumed() - _check_ffi_operation_result(new_ptr, - Reader._ERROR_MESSAGES[ - 'fragment_error' - ].format("Unknown error")) - self._swap_handle(new_ptr) + ), + Reader._ERROR_MESSAGES['fragment_error']) # Invalidate caches: processing a new BMFF fragment updates the native # reader's state, which can change the manifest data it returns. @@ -2616,7 +2683,6 @@ def with_fragment(self, format: str, stream, return self - def json(self) -> str: """Get the manifest store as a JSON string. @@ -2883,10 +2949,6 @@ def from_info(cls, signer_info: C2paSignerInfo) -> 'Signer': Raises: C2paError: If there was an error creating the signer """ - # Native libs plumbing: - # Clear any stale error state from previous operations - _clear_error_state() - signer_ptr = _lib.c2pa_signer_from_info(ctypes.byref(signer_info)) _check_ffi_operation_result( @@ -3004,10 +3066,6 @@ def wrapped_callback( cls._ERROR_MESSAGES['encoding_error'].format( str(e))) - # Native libs plumbing: - # Clear any stale error state from previous operations - _clear_error_state() - # Create the callback object using the callback function callback_cb = SignerCallback(wrapped_callback) @@ -3102,6 +3160,7 @@ class Builder(ManagedResource): 'archive_read_error': "Error loading ingredient from archive: {}", 'action_error': "Error adding action: {}", 'archive_error': "Error writing archive: {}", + 'archive_load_error': "Failed to load archive into builder: {}", 'sign_error': "Error during signing: {}", 'encoding_error': "Invalid UTF-8 characters in manifest: {}", 'json_error': "Failed to serialize manifest JSON: {}" @@ -3188,10 +3247,9 @@ def from_archive( ) try: - # A builder from an archive carries no context, which is what - # _init_attrs() already defaults to. + # A builder from an archive here carries no context. return cls._wrap_native_handle(handle) - except Exception: + except BaseException: # No instance took ownership, so the handle is still ours. ManagedResource._free_native_ptr(handle) raise @@ -3268,23 +3326,20 @@ def _init_from_context(self, context, json_str): 'builder_error' ].format("Unknown error") ) - except Exception: + except BaseException: if builder_ptr: ManagedResource._free_native_ptr(builder_ptr) raise - # Consume-and-return: builder_ptr is consumed (freed by the FFI even - # on failure), new_ptr is the valid pointer going forward. Nothing to - # free here on the error path. - new_ptr = _lib.c2pa_builder_with_definition(builder_ptr, json_str) + # Adopt the handle before the consuming call: _consume_and_swap needs + # an ACTIVE resource, and from here on normal cleanup owns the pointer + # whichever way the call goes. + self._activate(builder_ptr) - _check_ffi_operation_result(new_ptr, - Builder._ERROR_MESSAGES[ - 'builder_error' - ].format("Unknown error") - ) - - self._activate(new_ptr) + self._consume_and_swap( + lambda handle: _lib.c2pa_builder_with_definition( + handle, json_str), + Builder._ERROR_MESSAGES['builder_error']) def _init_attrs(self): super()._init_attrs() @@ -3567,22 +3622,10 @@ def with_archive(self, stream: Any) -> 'Builder': self._ensure_valid_state() with Stream(stream) as stream_obj: - try: - new_ptr = _lib.c2pa_builder_with_archive( - self._handle, stream_obj._stream) - except Exception as e: - self._mark_consumed() - raise C2paError( - f"Error loading archive: {e}" - ) - # c2pa_builder_with_archive consumed the old handle. A null return - # leaves no replacement to take ownership of: mark consumed, then - # raise. _swap_handle is only reached when new_ptr is non-null. - if not new_ptr: - self._mark_consumed() - _check_ffi_operation_result( - new_ptr, "Failed to load archive into builder") - self._swap_handle(new_ptr) + self._consume_and_swap( + lambda handle: _lib.c2pa_builder_with_archive( + handle, stream_obj._stream), + Builder._ERROR_MESSAGES['archive_load_error']) return self @@ -3645,9 +3688,11 @@ def _sign_internal( # Closing here ensures resources clean up, # and single use/single sign done by a Builder. self.close() - except Exception as e: + except BaseException as e: + # BaseException, not Exception: a KeyboardInterrupt mid-sign must + # still close the Builder rather than leaving its handle live. self.close() - raise C2paError(f"Error during signing: {e}") + raise C2paError(f"Error during signing: {e}") from e _check_ffi_operation_result( result, @@ -3863,8 +3908,6 @@ def format_embeddable(format: str, manifest_bytes: bytes) -> tuple[int, bytes]: Raises: C2paError: If there was an error converting the manifest """ - _clear_error_state() - format_str = format.encode('utf-8') manifest_array = (ctypes.c_ubyte * len(manifest_bytes)).from_buffer_copy( manifest_bytes @@ -3980,8 +4023,6 @@ def ed25519_sign(data: bytes, private_key: str) -> bytes: C2paError: If there was an error signing the data C2paError.Encoding: If the private key contains invalid UTF-8 chars """ - _clear_error_state() - if not data: raise C2paError("Data to sign cannot be empty") diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index 25c688a0..6166f6f6 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -9,6 +9,7 @@ Each function is called N times by run_profile.py. """ +import ctypes import gc import io import json @@ -27,6 +28,7 @@ Signer, Stream, ) +import c2pa.c2pa as c2pa_module FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" READING_FIXTURES_DIR = FIXTURES_DIR / "files-for-reading-tests" @@ -544,6 +546,148 @@ def scenario_builder_from_archive_roundtrip(iterations: int = 100) -> None: builder.sign(signer, "image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) +# Consume-and-return failure paths. +# +# These calls take ownership partway through their body, so a null return is +# ambiguous and getting it wrong leaks one handle per call. The success paths +# are covered above; these loop the failure paths, where the leak would be. + +def _untracked_reader_handle(): + """A pointer the native registry does not know about. + + A never-allocated buffer, so it is rejected like a stale handle without + allocating a real Reader per call, which would swamp the measurement. + Freed handles are unusable here: recycled addresses become tracked again. + """ + buf = ctypes.create_string_buffer(64) + return ctypes.cast(buf, ctypes.POINTER(c2pa_module.C2paReader)), buf + + +def scenario_reader_with_fragment_pre_consume_rejection( + iterations: int = 100) -> None: + """Loop the rejection that precedes the ownership transfer. + + The handle is still ours, so treating this as consumed drops a pointer + the registry still holds and leaked_bytes climbs with iterations. + """ + init_bytes = DASH_INIT_MP4.read_bytes() + fragment_bytes = DASH_FRAGMENT.read_bytes() + for _ in _iterate(iterations): + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + real_handle = reader._handle + # Keep the buffer alive: the cast pointer does not own it, and an + # early collection would hand the FFI a dangling address. + bogus, _buf = _untracked_reader_handle() + reader._handle = bogus + try: + reader.with_fragment("video/mp4", io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + raise AssertionError("pre-consume rejection did not raise") + except C2paError as e: + # Fail loudly: without these the scenario still runs when the + # ownership logic regresses, and a rejection that stops being + # recognised looks identical to a pass. + if not any(tag in str(e) for tag in + c2pa_module.ManagedResource._PRE_CONSUME_ERROR_TAGS): + raise AssertionError( + f"expected a pre-consume rejection, got: {e}") from e + if reader._handle is None: + raise AssertionError( + "handle was dropped on a pre-consume rejection; the " + "native side never took ownership, so this leaks") from e + finally: + # Restore before close() so the real handle is freed exactly once. + reader._handle = real_handle + reader.close() + + +def scenario_builder_with_archive_post_consume_failure( + iterations: int = 100) -> None: + """Loop a failure after the ownership transfer. + + The control: if the fix over-corrected into retaining handles the native + side already dropped, this scenario double-frees or leaks. + """ + for _ in _iterate(iterations): + builder = Builder(MANIFEST_BASE) + try: + builder.with_archive(io.BytesIO(b"not a valid archive")) + raise AssertionError("post-consume failure did not raise") + except C2paError: + pass + finally: + builder.close() + + +def scenario_with_fragment_marshalling_error(iterations: int = 100) -> None: + """Loop a failure that never reaches native code. + + Nothing was consumed, so the reader must stay usable. The old blanket + except marked it consumed here, leaking on what is only a type error. + """ + init_bytes = DASH_INIT_MP4.read_bytes() + for _ in _iterate(iterations): + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + try: + reader.with_fragment("video/mp4", object(), object()) + raise AssertionError("marshalling error did not raise") + except (C2paError, TypeError, ctypes.ArgumentError): + pass + finally: + # Must still be usable: nothing was handed over. + reader.json() + reader.close() + + +def scenario_with_fragment_mixed_outcomes(iterations: int = 100) -> None: + """Interleave success, pre-consume rejection and post-consume failure. + + Each path leaves a different state behind, so running them in sequence + catches a stale error being read as the current call's. + """ + init_bytes = DASH_INIT_MP4.read_bytes() + fragment_bytes = DASH_FRAGMENT.read_bytes() + for i in _iterate(iterations): + phase = i % 3 + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + try: + if phase == 0: + reader.with_fragment("video/mp4", io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + elif phase == 1: + real_handle = reader._handle + bogus, _buf = _untracked_reader_handle() + reader._handle = bogus + try: + reader.with_fragment("video/mp4", io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + raise AssertionError( + "pre-consume rejection did not raise") + except C2paError as e: + # A stale error from the phase before must not be read as + # this call's; the rejection would stop being recognised. + if not any( + tag in str(e) for tag in c2pa_module + .ManagedResource._PRE_CONSUME_ERROR_TAGS): + raise AssertionError( + f"expected a pre-consume rejection, got: {e}" + ) from e + if reader._handle is None: + raise AssertionError( + "handle dropped on a pre-consume rejection" + ) from e + finally: + reader._handle = real_handle + else: + try: + reader.with_fragment("video/mp4", io.BytesIO(b"garbage"), + io.BytesIO(b"garbage")) + except C2paError: + pass + finally: + reader.close() + + # Archive scenarios: builder as working store (to_archive/with_archive) and # per-ingredient archives (write_ingredient_archive/add_ingredient_from_archive). @@ -745,6 +889,64 @@ def scenario_reader_string_apis(iterations: int = 100) -> None: reader.close() +def scenario_builder_from_context_construction(iterations: int = 100) -> None: + """Loop Builder(context=...) construction, the consume-and-swap path. + + c2pa_builder_from_context hands back a handle that + c2pa_builder_with_definition then consumes and replaces. The Builder + adopts the first handle before that call so _consume_and_swap can own the + swap, which means a mis-sequenced swap leaks the replacement or frees the + consumed pointer twice. The other context builder scenarios sign a full + asset per iteration, so a one-handle regression here would sit under their + noise. This one only constructs and closes. + + scenario_reader_manifest_data_context is the Reader-side equivalent. + """ + context = Context() + for _ in _iterate(iterations): + builder = Builder(MANIFEST_BASE, context=context) + builder.close() + + +def scenario_sign_interrupted(iterations: int = 100) -> None: + """Loop a sign that is interrupted partway through the FFI call. + + _sign_internal catches BaseException so the Builder is closed even when + the interrupt is not an Exception subclass. Narrowing that back to + Exception leaks the whole Builder per iteration, since close() is skipped + and the handle outlives the object. + """ + source_bytes = SOURCE_JPEG.read_bytes() + signer = _make_signer() + real_sign = c2pa_module._lib.c2pa_builder_sign + + def _interrupt(*args): + raise KeyboardInterrupt + + c2pa_module._lib.c2pa_builder_sign = _interrupt + try: + for _ in _iterate(iterations): + builder = Builder(MANIFEST_BASE) + try: + builder.sign(signer, "image/jpeg", + io.BytesIO(source_bytes), io.BytesIO()) + except C2paError: + # The wrapper re-raises the interrupt as C2paError, having + # closed the Builder first. Anything else means the handler + # stopped catching it. + pass + else: + raise AssertionError("interrupted sign did not raise") + if (builder._lifecycle_state + is not c2pa_module.LifecycleState.CLOSED): + raise AssertionError( + "interrupted sign left the Builder open; its handle " + "leaks once per iteration") + finally: + c2pa_module._lib.c2pa_builder_sign = real_sign + signer.close() + + def scenario_signer_construction(iterations: int = 100) -> None: """Loop Signer.from_info()/__init__ construction and teardown. @@ -1207,6 +1409,13 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "builder_from_archive_roundtrip": scenario_builder_from_archive_roundtrip, "builder_with_archive_swap": scenario_builder_with_archive_swap, "reader_with_fragment_swap": scenario_reader_with_fragment_swap, + "with_fragment_pre_consume_rejection": + scenario_reader_with_fragment_pre_consume_rejection, + "with_archive_post_consume_failure": + scenario_builder_with_archive_post_consume_failure, + "with_fragment_marshalling_error": + scenario_with_fragment_marshalling_error, + "with_fragment_mixed_outcomes": scenario_with_fragment_mixed_outcomes, "builder_to_archive_with_ingredient": scenario_builder_to_archive_with_ingredient, "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": scenario_builder_sign_jpeg_archive_roundtrip_ingredient_in_archive, "builder_write_ingredient_archive": scenario_builder_write_ingredient_archive, @@ -1217,6 +1426,9 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "builder_error_invalid_manifest": scenario_builder_error_invalid_manifest, "reader_string_apis": scenario_reader_string_apis, "signer_construction": scenario_signer_construction, + "builder_from_context_construction": + scenario_builder_from_context_construction, + "sign_interrupted": scenario_sign_interrupted, "fork_reader_collect": scenario_fork_reader_collect, "fork_contended_mutex": scenario_fork_contended_mutex, "fork_thread_local_orphan": scenario_fork_thread_local_orphan, diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 466d5a81..0056789a 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -7688,6 +7688,10 @@ def test_reader_with_fragment_null_return_consumes_self(self): reader = Reader("video/mp4", init) consumed_handle = reader._handle + # The mock sets no error, which no native path does. Plant an + # operation-style one so a stale tag from another test is not read. + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked null return") + real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( lambda r, f, s, frag: None) @@ -7739,6 +7743,408 @@ def _raise(*_args): self.assertEqual(self._free_count(freed, consumed_handle), 0, "close() freed a handle the FFI already consumed") + # Consume-and-return ownership: the native call takes the handle partway + # through its body, so a null return does not say on its own whether the + # handle was consumed. These pin the classification down. + + @staticmethod + def _is_pre_consume_rejection(error_message): + """True if this native error means ownership never transferred.""" + if not error_message: + return False + return any(tag in error_message + for tag in ManagedResource._PRE_CONSUME_ERROR_TAGS) + + def _stale_reader_handle(self): + """A freed, untracked pointer, captured before close() nulls it. + + Take a fresh one per call: recycled addresses become tracked again. + """ + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + with open(init_path, "rb") as init: + victim = Reader("video/mp4", init) + stale = ctypes.cast(victim._handle, + ctypes.POINTER(c2pa_module.C2paReader)) + victim.close() + return stale + + @staticmethod + def _untracked_reader_handle(): + """A pointer the native registry never handed out. + + Rejected like a stale handle, but not a freed address, so it cannot + be recycled and start passing the registry lookup. + """ + buf = ctypes.create_string_buffer(64) + return (ctypes.cast(buf, ctypes.POINTER(c2pa_module.C2paReader)), + buf) + + def test_with_fragment_pre_consume_rejection_keeps_handle(self): + # Rejected before Box::from_raw, so nothing was consumed and the + # handle is still ours. Dropping it here would leak it. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + + reader._handle = self._stale_reader_handle() + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error) as caught: + reader.with_fragment("video/mp4", init, frag) + finally: + reader._handle = real_handle + + self.assertIn("UntrackedPointer", str(caught.exception)) + # Ownership never transferred, so the resource stays usable. + self.assertIsNotNone(reader._handle) + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + self.assertTrue(reader.json()) + reader.close() + + def test_with_fragment_pre_consume_rejection_does_not_leak(self): + # A handle dropped on this path leaks one reader per call. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + + for _ in range(25): + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + reader._handle = self._stale_reader_handle() + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("video/mp4", init, frag) + finally: + reader._handle = real_handle + # Still owns a working handle every time round, so nothing + # leaked: a dropped handle would leave the reader closed. + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + self.assertTrue(reader.json()) + reader.close() + + def test_with_archive_post_consume_failure_consumes_handle(self): + # Ownership taken, then the operation failed: the handle is gone, + # so close() must not free it again. + builder = Builder(json.dumps( + {"claim_generator_info": [{"name": "test", "version": "0.1"}], + "assertions": []})) + consumed_handle = builder._handle + + with self.assertRaises(Error): + builder.with_archive(io.BytesIO(b"not a valid archive")) + + self.assertIsNone(builder._handle) + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) + + freed = self._instrument_frees() + builder.close() + self.assertEqual(self._free_count(freed, consumed_handle), 0, + "close() freed a handle the FFI already consumed") + + def test_with_fragment_marshalling_error_keeps_handle(self): + # Never reaches native code, so nothing was consumed. The old + # blanket except marked it consumed here and leaked the handle. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + + def _bad_marshalling(*_args): + raise ctypes.ArgumentError("wrong argument type") + + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = _bad_marshalling + try: + with open(init_path, "rb") as init, \ + open(init_path, "rb") as frag: + with self.assertRaises(ctypes.ArgumentError): + reader.with_fragment("video/mp4", init, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + + self.assertIs(reader._handle, real_handle) + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + self.assertTrue(reader.json(), + "a marshalling failure never reached the FFI, so " + "the reader must still be usable") + + reader.close() + + def test_unknown_failure_drops_handle_without_freeing(self): + # Ownership unknowable, so the handle is let go rather than freed. + # Needs a mock: every real null return sets an error. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + + consumed_handle = reader._handle + # Not a pre-consume tag, so the mocked failure reads as unplaceable. + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked null return") + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = ( + lambda r, f, s, frag: None) + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("video/mp4", init, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + + # Unplaceable, so the handle is let go rather than freed twice. + self.assertIsNone(reader._handle) + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + + freed = self._instrument_frees() + reader.close() + self.assertEqual(self._free_count(freed, consumed_handle), 0, + "close() freed a handle of unknown ownership") + + def test_pre_consume_rejection_is_typed(self): + # Rejections arrive wrapped as "Other: UntrackedPointer: 0x...". + self.assertTrue(self._is_pre_consume_rejection( + "Other: UntrackedPointer: 0x600001234567")) + self.assertTrue(self._is_pre_consume_rejection( + "Other: WrongPointerType: 0x600001234567")) + self.assertFalse(self._is_pre_consume_rejection( + "Verify: invalid JUMBF header")) + self.assertFalse(self._is_pre_consume_rejection(None)) + self.assertFalse(self._is_pre_consume_rejection("")) + + def test_pre_consume_tags_still_match_the_native_wording(self): + # Classification keys on error text (the numeric code is not + # exported), so a native rename would silently misjudge ownership. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + + reader._handle = self._stale_reader_handle() + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error) as caught: + reader.with_fragment("video/mp4", init, frag) + finally: + reader._handle = real_handle + + message = str(caught.exception) + self.assertTrue( + self._is_pre_consume_rejection(message), + f"the native rejection wording changed and no longer matches " + f"_PRE_CONSUME_ERROR_TAGS; ownership will be misjudged: " + f"{message!r}") + reader.close() + + def test_stale_handle_is_actually_rejected_every_time(self): + # A handle that stopped being rejected would quietly measure the + # success path. Uses the never-allocated buffer: freed addresses get + # recycled and start passing the registry lookup. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + + for _ in range(25): + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + bogus, _buf = self._untracked_reader_handle() + reader._handle = bogus + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error) as caught: + reader.with_fragment("video/mp4", init, frag) + self.assertTrue( + self._is_pre_consume_rejection( + str(caught.exception)), + "the bogus handle was not rejected, so this stopped " + "exercising the pre-consume path") + finally: + reader._handle = real_handle + reader.close() + + def test_perf_scenario_bogus_handle_is_rejected(self): + # The perf scenarios use a plain buffer so looping does not swamp + # the measurement. It still has to produce a real rejection. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + + bogus, _buf = self._untracked_reader_handle() + reader._handle = bogus + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error) as caught: + reader.with_fragment("video/mp4", init, frag) + finally: + reader._handle = real_handle + + self.assertTrue( + self._is_pre_consume_rejection(str(caught.exception)), + "the perf scenarios' bogus handle is no longer rejected, so " + "with_fragment_pre_consume_rejection measures nothing") + # Handle kept, so the reader still works and frees normally. + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + self.assertTrue(reader.json()) + reader.close() + + def test_every_null_return_sets_its_own_error(self): + # Reading the slot without clearing it is only sound because every + # null return sets an error. Check each path reports its own. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + + # Leave a recognisable error behind, so anything stale shows up. + try: + Reader("image/jpeg", io.BytesIO(b"not an image")).json() + except Error: + pass + self.assertIn("NotSupported", c2pa_module._read_native_error() or "") + + # Pre-consume rejection: reports UntrackedPointer, not NotSupported. + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + real_handle = reader._handle + reader._handle = self._stale_reader_handle() + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error) as caught: + reader.with_fragment("video/mp4", init, frag) + finally: + reader._handle = real_handle + self.assertIn("UntrackedPointer", str(caught.exception)) + self.assertNotIn("NotSupported", str(caught.exception)) + reader.close() + + # Post-consume failure: reports the operation error, not the + # UntrackedPointer left by the step above. + builder = Builder(json.dumps( + {"claim_generator_info": [{"name": "test", "version": "0.1"}], + "assertions": []})) + with self.assertRaises(Error) as caught: + builder.with_archive(io.BytesIO(b"not a valid archive")) + self.assertNotIn("UntrackedPointer", str(caught.exception)) + builder.close() + + def test_pre_consume_classification_holds_across_threads(self): + # The error slot is thread-local, so concurrent calls do not mask + # each other's errors and misjudge ownership. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + init_bytes = open(init_path, "rb").read() + frag_bytes = open(fragment_path, "rb").read() + problems = [] + + def worker(): + for _ in range(10): + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + real_handle = reader._handle + # A never-allocated buffer, not a freed pointer: with several + # threads churning the allocator, a freed address gets reused + # and starts passing the registry lookup, which would end the + # iteration in a real consume instead of a rejection. + bogus, _buf = self._untracked_reader_handle() + reader._handle = bogus + try: + reader.with_fragment("video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(frag_bytes)) + problems.append("rejection did not raise") + except Error as e: + if not self._is_pre_consume_rejection(str(e)): + problems.append(f"misclassified: {str(e)[:60]}") + finally: + reader._handle = real_handle + if reader._lifecycle_state != LifecycleState.ACTIVE: + problems.append("handle dropped on a rejection") + reader.close() + + threads = [threading.Thread(target=worker) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(problems, [], + "ownership was misjudged under concurrency") + + def test_reading_the_native_error_does_not_empty_the_slot(self): + # c2pa_error() peeks, so nothing Python can call empties the slot. + # _consume_and_swap depends on this. + try: + Reader("image/jpeg", io.BytesIO(b"not an image")).json() + except Error: + pass + + first = c2pa_module._read_native_error() + self.assertTrue(first, "expected a native error to have been set") + + self.assertEqual( + c2pa_module._read_native_error(), first, + "reading emptied the native slot; the comments in " + "_consume_and_swap about a persistent error are now wrong") + + def test_read_native_error_returns_none_for_an_empty_message(self): + # c2pa_error() returns an owned pointer to "" when no error is set, + # never NULL, so the pointer cannot be the "is there an error" test. + original = c2pa_module._lib.c2pa_error + empty = ctypes.create_string_buffer(b"") + + try: + c2pa_module._lib.c2pa_error = lambda: ctypes.cast( + empty, ctypes.c_void_p).value + self.assertIsNone( + c2pa_module._read_native_error(), + "an empty native message must read as None, otherwise " + "callers' 'if error:' checks are only accidentally right") + finally: + c2pa_module._lib.c2pa_error = original + + def test_mocked_null_without_error_is_a_known_limitation(self): + # A null with no error of its own is the case that breaks: the slot + # still holds whatever came before. No native path does this, so it + # is pinned here rather than defended in _consume_and_swap. + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + + c2pa_module._lib.c2pa_error_set_last( + b"UntrackedPointer: 0xdeadbeef") + + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + + real_call = c2pa_module._lib.c2pa_reader_with_fragment + c2pa_module._lib.c2pa_reader_with_fragment = ( + lambda r, f, s, frag: None) + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + with self.assertRaises(Error): + reader.with_fragment("video/mp4", init, frag) + finally: + c2pa_module._lib.c2pa_reader_with_fragment = real_call + # Nothing clears the slot, so a planted tag would follow other + # tests around and change how their failures are classified. + c2pa_module._lib.c2pa_error_set_last( + b"Other: cleared by test teardown") + + # The stale tag wins, so the handle is kept. Safe here (the mock + # consumed nothing), and the reader is still usable. + self.assertIsNotNone(reader._handle) + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + reader.close() + # Backfilling a pointer minted by a direct FFI call. Builder.from_archive # is the only production caller of _wrap_native_handle, so these are the # only tests that drive the primitive as the generic entry point it is. @@ -8002,6 +8408,252 @@ def _boom(*args, **kwargs): self.assertEqual(len(freed), 1, "from_archive leaked the handle when the wrap failed") + # Interrupt paths. + # + # These handlers catch BaseException rather than Exception, so a + # KeyboardInterrupt arriving mid-call still runs the cleanup. Catching + # Exception would let the interrupt escape past the free/close. + + def test_interrupted_context_build_frees_builder_ptr(self): + # An interrupt between c2pa_context_builder_new and the build call + # leaves builder_ptr owned by nobody but this frame. + freed = self._instrument_frees() + real_build = c2pa_module._lib.c2pa_context_builder_build + + def _interrupt(ptr): + raise KeyboardInterrupt + + c2pa_module._lib.c2pa_context_builder_build = _interrupt + try: + with self.assertRaises(KeyboardInterrupt): + Context(signer=self._ctx_make_signer()) + finally: + c2pa_module._lib.c2pa_context_builder_build = real_build + + self.assertEqual(len(freed), 1, + "interrupted Context build leaked the context " + "builder pointer") + + def test_interrupted_sign_closes_builder(self): + # sign() borrows the Builder and closes it afterwards. An interrupt + # must not skip that close, or the handle outlives the object. + builder = Builder(self.test_manifest) + signer = self._ctx_make_signer() + self.addCleanup(signer.close) + with open(os.path.join(FIXTURES_DIR, "C.jpg"), "rb") as f: + source_bytes = f.read() + + real_sign = c2pa_module._lib.c2pa_builder_sign + + def _interrupt(*args): + raise KeyboardInterrupt + + c2pa_module._lib.c2pa_builder_sign = _interrupt + try: + # With `except Exception` the KeyboardInterrupt escapes unhandled + # and self.close() never runs; the BaseException handler catches + # it and re-raises it wrapped, having closed the Builder first. + with self.assertRaises(Error): + builder.sign(signer, "image/jpeg", + io.BytesIO(source_bytes), io.BytesIO()) + finally: + c2pa_module._lib.c2pa_builder_sign = real_sign + + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED, + "interrupted sign left the Builder open") + self.assertIsNone(builder._handle) + + def test_sign_failure_chains_the_original_exception(self): + # The wrapper re-raises as C2paError; losing __cause__ hides which + # call actually failed. + builder = Builder(self.test_manifest) + signer = self._ctx_make_signer() + self.addCleanup(signer.close) + + sentinel = RuntimeError("native call blew up") + real_sign = c2pa_module._lib.c2pa_builder_sign + + def _boom(*args): + raise sentinel + + c2pa_module._lib.c2pa_builder_sign = _boom + try: + with self.assertRaises(Error) as ctx: + builder.sign(signer, "image/jpeg", + io.BytesIO(b"x"), io.BytesIO()) + finally: + c2pa_module._lib.c2pa_builder_sign = real_sign + + self.assertIs(ctx.exception.__cause__, sentinel, + "signing error dropped the original exception") + + def test_marshalling_error_on_set_signer_leaves_signer_owned(self): + # ctypes.ArgumentError means the call never reached the native side, + # so the signer was never taken and must keep owning its handle. + # + # This passes with or without the explicit ArgumentError re-raise at + # the call site, because a bare re-raise matches what happens with no + # handler at all. It pins the invariant against a future edit + # that adds work between the FFI call and _mark_consumed(), which would + # otherwise start marking an untaken signer as consumed. + signer = self._ctx_make_signer() + self.addCleanup(signer.close) + real_set = c2pa_module._lib.c2pa_context_builder_set_signer + + def _bad_marshal(builder_ptr, signer_ptr): + raise ctypes.ArgumentError("bad argument type") + + c2pa_module._lib.c2pa_context_builder_set_signer = _bad_marshal + try: + with self.assertRaises(ctypes.ArgumentError): + Context(signer=signer) + finally: + c2pa_module._lib.c2pa_context_builder_set_signer = real_set + + self.assertIsNotNone( + signer._handle, + "a signer the native side never took was marked consumed") + self.assertEqual(signer._lifecycle_state, LifecycleState.ACTIVE) + + +class TestErrorPlumbing(unittest.TestCase): + """Covers the error helpers themselves, which had no direct tests.""" + + def _set_native_error(self, text): + c2pa_module._lib.c2pa_error_set_last(text.encode('utf-8')) + + def test_every_error_tag_maps_to_its_typed_subclass(self): + # The wire text is "Tag: message"; each tag gets its own subclass so + # callers can catch precisely. + tags = [ + "Assertion", "AssertionNotFound", "Decoding", "Encoding", + "FileNotFound", "Io", "Json", "Manifest", "ManifestNotFound", + "NotSupported", "Other", "RemoteManifest", "ResourceNotFound", + "Signature", "Verify", "UntrackedPointer", "WrongPointerType", + ] + for tag in tags: + with self.subTest(tag=tag): + expected = getattr(Error, tag) + with self.assertRaises(expected): + c2pa_module._raise_typed_c2pa_error(f"{tag}: detail") + + def test_unmapped_tag_falls_back_to_base_error(self): + with self.assertRaises(Error) as ctx: + c2pa_module._raise_typed_c2pa_error("Nonsense: detail") + # Base class only: no subclass should claim an unknown tag. + self.assertIs(type(ctx.exception), Error) + + def test_check_ffi_operation_result_raises_with_native_message(self): + self._set_native_error("Io: disk exploded") + with self.assertRaises(Error) as ctx: + c2pa_module._check_ffi_operation_result(None, "fallback text") + self.assertIn("disk exploded", str(ctx.exception)) + + def test_check_ffi_operation_result_uses_fallback_when_slot_empty(self): + # The slot is sticky and thread-local, so a fresh thread is the only + # way to observe it unset. + captured = [] + + def run(): + try: + c2pa_module._check_ffi_operation_result(None, "fallback text") + except BaseException as e: # noqa: BLE001 - reported below + captured.append(e) + + t = threading.Thread(target=run) + t.start() + t.join() + + self.assertEqual(len(captured), 1, "expected a raise on failure") + self.assertIsInstance(captured[0], Error) + self.assertIn("fallback text", str(captured[0])) + + def test_check_ffi_operation_result_passes_success_through(self): + self.assertEqual( + c2pa_module._check_ffi_operation_result(42, "unused"), 42) + + def test_stream_creation_failure_reports_a_real_message(self): + # Regression: used to raise bare Exception("...: None"), because + # _parse_operation_result_for_error never returns a message. + real = c2pa_module._lib.c2pa_create_stream + c2pa_module._lib.c2pa_create_stream = lambda *a: None + try: + # With an error set, the message must carry it. + self._set_native_error("Io: stream refused") + with self.assertRaises(Error) as ctx: + c2pa_module.Stream(io.BytesIO(b"x")) + self.assertIn("stream refused", str(ctx.exception)) + + # With the slot unset, the old code raised bare Exception. + captured = [] + + def run(): + try: + c2pa_module.Stream(io.BytesIO(b"x")) + except BaseException as e: # noqa: BLE001 - reported below + captured.append(e) + + t = threading.Thread(target=run) + t.start() + t.join() + + self.assertEqual(len(captured), 1, "expected a raise on failure") + self.assertIsInstance( + captured[0], Error, + "stream failure must raise C2paError, not bare Exception") + self.assertNotIn("None", str(captured[0])) + finally: + c2pa_module._lib.c2pa_create_stream = real + + def test_supported_mime_types_raises_instead_of_returning_empty(self): + # Regression: `if error:` was always False, so a native failure + # returned an empty list instead of raising. Only visible with the + # slot unset, hence the fresh thread. + captured = [] + + def run(): + try: + captured.append( + c2pa_module._get_supported_mime_types( + lambda count: None, None)) + except BaseException as e: # noqa: BLE001 - reported below + captured.append(e) + + t = threading.Thread(target=run) + t.start() + t.join() + + self.assertIsInstance( + captured[0], Error, + "a failed MIME lookup returned data instead of raising") + + def test_supported_mime_types_reports_the_native_message(self): + self._set_native_error("Io: mime lookup failed") + with self.assertRaises(Error) as ctx: + c2pa_module._get_supported_mime_types(lambda count: None, None) + self.assertIn("mime lookup failed", str(ctx.exception)) + + +class TestErrorsStillRaiseAfterCleanup(unittest.TestCase): + """Each surface that lost a _clear_error_state() call still reports.""" + + def test_reader_on_garbage_raises(self): + with self.assertRaises(Error): + Reader("image/jpeg", io.BytesIO(b"not an image")).json() + + def test_signer_from_info_with_bad_certs_raises(self): + with self.assertRaises(Error): + c2pa_module.create_signer_from_info(C2paSignerInfo( + alg=b"es256", + sign_cert=b"not a certificate", + private_key=b"not a key", + ta_url=b"", + )) + + def test_ed25519_sign_with_empty_data_raises(self): + with self.assertRaises(Error): + c2pa_module.ed25519_sign(b"", "not a key") + if __name__ == '__main__': unittest.main(warnings='ignore') From c24ede6d0b7dc12a52416b216e27a7391a6902a6 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:25:21 -0700 Subject: [PATCH 15/67] fix: Refactor --- src/c2pa/c2pa.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index ee06d265..7484af7d 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -493,9 +493,6 @@ def _cleanup_resources(self): and self._lifecycle_state != LifecycleState.CLOSED ): self._lifecycle_state = LifecycleState.CLOSED - # A failing _release() must not skip the free below: - # that would strand the native handle on an object already - # marked CLOSED, making it unreachable and unfreeable. self._safe_release() if hasattr(self, '_handle') and self._handle: try: @@ -1691,8 +1688,7 @@ def __init__( if signer is not None: signer._ensure_valid_state() # c2pa_context_builder_set_signer takes ownership of the - # signer pointer immediately (Box::from_raw), on its error - # path as well as on success. + # signer pointer , on its error path as well as on success. self._signer_callback_cb = signer._callback_cb try: result = ( @@ -2087,11 +2083,6 @@ def close(self): if self._closed: return if is_foreign_process(self): - # Unlike ManagedResource, which leaves a child's copy active - # because the parent still owns the pointer, a Stream's callbacks - # are bound to the parent's objects. - # The child's copy can never be used, so mark it closed - # and let the parent free it. self._closed = True self._initialized = False return From 359ce9862fcf3dee33ec84aacc1717d34c35352e Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:35:21 -0700 Subject: [PATCH 16/67] fix: Refactor 3 --- src/c2pa/c2pa.py | 51 +++++++----------------- tests/perf/scenarios.py | 40 ------------------- tests/test_unit_tests.py | 83 ---------------------------------------- 3 files changed, 14 insertions(+), 160 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 7484af7d..1b020ee2 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -302,16 +302,11 @@ def _safe_release(self): ) def _mark_consumed(self): - """Mark as consumed by an FFI call that took ownership of the native - handle. - - Ownership contract: the native pointer is now owned elsewhere, so this - does not free it. It releases only Python-side resources (via - `_release()`: callback refs, streams, caches) and marks the resource - closed. Marking it closed stops `_cleanup_resources` from freeing the - now foreign-owned pointer later; releasing here means those Python - resources are not stranded until garbage collection. + """Mark as consumed by an FFI call that took ownership + of native resources e.g. pointers. This means we should not + call clean-up here anymore, and leave it to the new owner. """ + if is_foreign_process(self): self._handle = None self._lifecycle_state = LifecycleState.CLOSED @@ -326,10 +321,7 @@ def _mark_consumed(self): def _activate(self, handle): """Attach a native handle to self and mark it active. - - Ownership of `handle` transfers here: this object frees it on close. - Only an uninitialized resource can be activated, so a handle can never - be activated twice and a closed resource can never be reopened. + Ownership of `handle` transfers here. Args: handle: Non-null native pointer to take ownership of @@ -353,16 +345,9 @@ def _activate(self, handle): def _swap_handle(self, new_handle): """Replace the handle after an FFI call consumed the old one and returned a replacement. - - Ownership contract: the caller guarantees the callee has already - consumed and freed the old pointer. This method never frees the old - pointer; it only rebinds `self._handle` to the replacement. Calling it - when the old pointer was not consumed leaks that pointer. - A null return from such a call is ambiguous (the callee may have failed validation before taking ownership, or failed the operation after), so callers must not call this with a null replacement. - Requires the resource to be active. Args: @@ -389,15 +374,11 @@ def _swap_handle(self, new_handle): def _consume_and_swap(self, ffi_call, error_message): """Run an FFI call that consumes this handle and returns a replacement. - The native lib takes ownership partway through the call, - so a null return can be ambiguous. - The native error tells the cases apart: + The native lib may take ownership partway through the call. + The native error tells if ownership was transferred: - a pointer rejection (`_PRE_CONSUME_ERROR_TAGS`) precedes the transfer, so the handle is still ours and is kept; - - any other error means it was taken, then the operation failed; - - a null with no error cannot be placed, so the handle is dropped - anyway: leaking is recoverable, double-freeing is not. No native - path does this, so it only appears under mocks. + - any other error means it was taken, then the operation failed. The error is read without clearing it first. The slot is sticky: c2pa_error() peeks and nothing empties it. @@ -429,8 +410,8 @@ def _consume_and_swap(self, ffi_call, error_message): if error: if any(tag in error for tag in ManagedResource._PRE_CONSUME_ERROR_TAGS): - # Rejected before ownership transferred, so the handle is - # still ours: keep it and let normal cleanup free it. + # Rejected before ownership transferred, + # so the handle is still ours. logger.warning( "%s: native call rejected the handle before taking " "ownership (%s); handle retained", @@ -438,7 +419,7 @@ def _consume_and_swap(self, ffi_call, error_message): error) _raise_typed_c2pa_error(error) - # Ownership transferred and then the operation failed. + # Ownership transferred and then an operation failed. self._mark_consumed() _raise_typed_c2pa_error(error) @@ -448,14 +429,11 @@ def _consume_and_swap(self, ffi_call, error_message): @classmethod def _wrap_native_handle(cls, handle): """Build a brand-new instance around an already-valid, - already-owned native handle, bypassing __init__ entirely. - - __init__ is bypassed, so `_init_attrs()` supplies the class's own - defaults and the instance is stamped with the creating process. + already-owned native handle (bypassing __init__). Everything an instance needs besides the native handle must be set in - `_init_attrs()`, not `__init__` — `_wrap_native_handle` never runs - `__init__`. An attribute a subclass sets only in `__init__` will be + `_init_attrs()`. `_wrap_native_handle` never runs `__init__`. + An attribute a subclass sets only in `__init__` will be missing (or left at its `_init_attrs` default) on a wrapped instance. Ownership of `handle` transfers only on successful return. If this @@ -468,7 +446,6 @@ def _wrap_native_handle(cls, handle): C2paError: If the handle is null """ obj = object.__new__(cls) - # Stamps _owner_pid, which the fork guard relies on. ManagedResource.__init__(obj) obj._init_attrs() obj._activate(handle) diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index 6166f6f6..23300aed 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -908,45 +908,6 @@ def scenario_builder_from_context_construction(iterations: int = 100) -> None: builder.close() -def scenario_sign_interrupted(iterations: int = 100) -> None: - """Loop a sign that is interrupted partway through the FFI call. - - _sign_internal catches BaseException so the Builder is closed even when - the interrupt is not an Exception subclass. Narrowing that back to - Exception leaks the whole Builder per iteration, since close() is skipped - and the handle outlives the object. - """ - source_bytes = SOURCE_JPEG.read_bytes() - signer = _make_signer() - real_sign = c2pa_module._lib.c2pa_builder_sign - - def _interrupt(*args): - raise KeyboardInterrupt - - c2pa_module._lib.c2pa_builder_sign = _interrupt - try: - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE) - try: - builder.sign(signer, "image/jpeg", - io.BytesIO(source_bytes), io.BytesIO()) - except C2paError: - # The wrapper re-raises the interrupt as C2paError, having - # closed the Builder first. Anything else means the handler - # stopped catching it. - pass - else: - raise AssertionError("interrupted sign did not raise") - if (builder._lifecycle_state - is not c2pa_module.LifecycleState.CLOSED): - raise AssertionError( - "interrupted sign left the Builder open; its handle " - "leaks once per iteration") - finally: - c2pa_module._lib.c2pa_builder_sign = real_sign - signer.close() - - def scenario_signer_construction(iterations: int = 100) -> None: """Loop Signer.from_info()/__init__ construction and teardown. @@ -1428,7 +1389,6 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "signer_construction": scenario_signer_construction, "builder_from_context_construction": scenario_builder_from_context_construction, - "sign_interrupted": scenario_sign_interrupted, "fork_reader_collect": scenario_fork_reader_collect, "fork_contended_mutex": scenario_fork_contended_mutex, "fork_thread_local_orphan": scenario_fork_thread_local_orphan, diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 0056789a..59a68ee8 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -8408,61 +8408,6 @@ def _boom(*args, **kwargs): self.assertEqual(len(freed), 1, "from_archive leaked the handle when the wrap failed") - # Interrupt paths. - # - # These handlers catch BaseException rather than Exception, so a - # KeyboardInterrupt arriving mid-call still runs the cleanup. Catching - # Exception would let the interrupt escape past the free/close. - - def test_interrupted_context_build_frees_builder_ptr(self): - # An interrupt between c2pa_context_builder_new and the build call - # leaves builder_ptr owned by nobody but this frame. - freed = self._instrument_frees() - real_build = c2pa_module._lib.c2pa_context_builder_build - - def _interrupt(ptr): - raise KeyboardInterrupt - - c2pa_module._lib.c2pa_context_builder_build = _interrupt - try: - with self.assertRaises(KeyboardInterrupt): - Context(signer=self._ctx_make_signer()) - finally: - c2pa_module._lib.c2pa_context_builder_build = real_build - - self.assertEqual(len(freed), 1, - "interrupted Context build leaked the context " - "builder pointer") - - def test_interrupted_sign_closes_builder(self): - # sign() borrows the Builder and closes it afterwards. An interrupt - # must not skip that close, or the handle outlives the object. - builder = Builder(self.test_manifest) - signer = self._ctx_make_signer() - self.addCleanup(signer.close) - with open(os.path.join(FIXTURES_DIR, "C.jpg"), "rb") as f: - source_bytes = f.read() - - real_sign = c2pa_module._lib.c2pa_builder_sign - - def _interrupt(*args): - raise KeyboardInterrupt - - c2pa_module._lib.c2pa_builder_sign = _interrupt - try: - # With `except Exception` the KeyboardInterrupt escapes unhandled - # and self.close() never runs; the BaseException handler catches - # it and re-raises it wrapped, having closed the Builder first. - with self.assertRaises(Error): - builder.sign(signer, "image/jpeg", - io.BytesIO(source_bytes), io.BytesIO()) - finally: - c2pa_module._lib.c2pa_builder_sign = real_sign - - self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED, - "interrupted sign left the Builder open") - self.assertIsNone(builder._handle) - def test_sign_failure_chains_the_original_exception(self): # The wrapper re-raises as C2paError; losing __cause__ hides which # call actually failed. @@ -8487,34 +8432,6 @@ def _boom(*args): self.assertIs(ctx.exception.__cause__, sentinel, "signing error dropped the original exception") - def test_marshalling_error_on_set_signer_leaves_signer_owned(self): - # ctypes.ArgumentError means the call never reached the native side, - # so the signer was never taken and must keep owning its handle. - # - # This passes with or without the explicit ArgumentError re-raise at - # the call site, because a bare re-raise matches what happens with no - # handler at all. It pins the invariant against a future edit - # that adds work between the FFI call and _mark_consumed(), which would - # otherwise start marking an untaken signer as consumed. - signer = self._ctx_make_signer() - self.addCleanup(signer.close) - real_set = c2pa_module._lib.c2pa_context_builder_set_signer - - def _bad_marshal(builder_ptr, signer_ptr): - raise ctypes.ArgumentError("bad argument type") - - c2pa_module._lib.c2pa_context_builder_set_signer = _bad_marshal - try: - with self.assertRaises(ctypes.ArgumentError): - Context(signer=signer) - finally: - c2pa_module._lib.c2pa_context_builder_set_signer = real_set - - self.assertIsNotNone( - signer._handle, - "a signer the native side never took was marked consumed") - self.assertEqual(signer._lifecycle_state, LifecycleState.ACTIVE) - class TestErrorPlumbing(unittest.TestCase): """Covers the error helpers themselves, which had no direct tests.""" From 85b6bfd973cf11eb8ff53e83f58ba7eb2509c659 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:21:17 -0700 Subject: [PATCH 17/67] fix: Refactor once more --- src/c2pa/c2pa.py | 9 +---- tests/test_unit_tests.py | 61 ++++++++++++------------------- tests/test_unit_tests_threaded.py | 50 +++---------------------- 3 files changed, 31 insertions(+), 89 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 1b020ee2..d631dbee 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -2502,7 +2502,7 @@ def _init_from_context(self, context, format_or_path, raise # Adopt the handle before the consuming call: _consume_and_swap - # needs an ACTIVE resource, and from here on normal cleanup owns + # needs an active resource, and from here on normal cleanup owns # the pointer whichever way the call goes. self._activate(reader_ptr) @@ -2572,9 +2572,6 @@ def _close_streams(self): def _release(self): """Release Reader-specific resources (caches, stream, backing file). - - Every teardown path runs this, including _mark_consumed(), which - close() never gets a chance to follow. """ self._manifest_json_str_cache = None self._manifest_data_cache = None @@ -3300,7 +3297,7 @@ def _init_from_context(self, context, json_str): raise # Adopt the handle before the consuming call: _consume_and_swap needs - # an ACTIVE resource, and from here on normal cleanup owns the pointer + # an active resource, and from here on normal cleanup owns the pointer # whichever way the call goes. self._activate(builder_ptr) @@ -3657,8 +3654,6 @@ def _sign_internal( # and single use/single sign done by a Builder. self.close() except BaseException as e: - # BaseException, not Exception: a KeyboardInterrupt mid-sign must - # still close the Builder rather than leaving its handle live. self.close() raise C2paError(f"Error during signing: {e}") from e diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 59a68ee8..f440d115 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -6935,8 +6935,8 @@ class TestManagedResourceLifecycle(unittest.TestCase): the ownership hand-offs between Python and the native library. setUp records frees instead of performing them, so a miscount reads as a - leak or a double-free rather than a crash. Tests holding real handles - call _use_real_frees() first. + leak or a double-free rather than a crash. + Tests holding real handles call _use_real_frees() first. """ class _FakeHandleResource(ManagedResource): @@ -6961,7 +6961,7 @@ def _release(self): self.release_calls += 1 class _ExtenderResource(ManagedResource): - """A downstream extender that owns a raw handle and wraps it via + """Am extender that owns a raw handle and wraps it via _wrap_native_handle. It carries several attributes of its own, all defaulted in _init_attrs (not __init__), and _release reads them, so a missing attribute would surface as an AttributeError on teardown. @@ -6994,7 +6994,7 @@ def _free_counts(self): return counts def _use_real_frees(self): - """Undo setUp's recorder, so native handles are really freed.""" + """Undo free recorder, so native handles are really freed.""" ManagedResource._free_native_ptr = self._real_free def _make_signer(self): @@ -7005,8 +7005,6 @@ def _make_signer(self): return Signer.from_info(C2paSignerInfo( b"es256", certs, key, b"http://timestamp.digicert.com")) - # _cleanup_resources: a failing _release() must not strand the handle - def test_release_failure_still_frees_handle(self): res = self._CallbackHoldingResource() # _callback_cb is never set, so _release() raises AttributeError. @@ -7029,8 +7027,6 @@ def test_release_failure_is_logged(self): any('Failed to release' in line for line in captured.output), f"_release() failure was not logged: {captured.output}") - # _activate guards - def test_activate_rejects_null_handle(self): res = self._FakeHandleResource() @@ -7078,16 +7074,14 @@ def test_activate_does_not_mutate_on_rejection(self): "rejected activation replaced the handle") self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) - # _swap_handle - def test_swap_handle_does_not_free_consumed_handle(self): res = self._FakeHandleResource() res._activate(0xAAA1) res._swap_handle(0xAAA2) - # The FFI already owns and frees the old pointer, so freeing it here - # would be a double-free. + # The FFI already owns and frees the old pointer, + # so freeing it here would be a double-free. self.assertEqual(self.freed, []) self.assertEqual(res._handle, 0xAAA2) @@ -7117,8 +7111,6 @@ def test_swap_handle_rejects_null_replacement(self): self.assertEqual(res._handle, 0x7777) self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) - # _wrap_native_handle - def test_wrap_native_handle_bypasses_init(self): seen = [] @@ -7138,7 +7130,6 @@ def _release(self): self.assertEqual(obj._tag, 'from _init_attrs') self.assertEqual(obj._lifecycle_state, LifecycleState.ACTIVE) self.assertTrue(obj.is_valid) - # ManagedResource.__init__ still ran, so fork-safety is intact. self.assertTrue(hasattr(obj, '_owner_pid')) obj.close() @@ -7191,8 +7182,7 @@ def test_foreign_child_skips_free_for_wrapped_and_swapped(self): "forked child freed a pointer its parent still owns") # The child must not free a pointer the parent still owns. # The child does mark its own copies closed and nulls their handles, - # which is safe (the parent holds a separate copy) and - # stops the child from reusing a parent-owned handle. + # which stops the child from reusing a parent-owned handle. self.assertEqual(wrapped._lifecycle_state, LifecycleState.CLOSED) self.assertIsNone(wrapped._handle) self.assertEqual(swapped._lifecycle_state, LifecycleState.CLOSED) @@ -7244,8 +7234,7 @@ def test_consumed_resource_frees_nothing_in_either_process(self): self.assertEqual(self.freed, []) # Consuming a handle hands the native pointer to a new owner, - # but the Python-side resources are still ours to let go of. - + # but the Python-side resources are still ours and we need to free. def test_mark_consumed_releases_python_resources(self): res = self._ReleaseRecordingResource() res._activate(0xF1) @@ -7282,7 +7271,7 @@ def test_extender_wraps_handle_fully_built(self): obj = self._ExtenderResource._wrap_native_handle(0xE0) # Every attribute _init_attrs defaults is present, - # even thoug __init__ never ran. + # even though __init__ never ran. self.assertEqual(obj.label, "extender") self.assertEqual(obj.buffer, []) self.assertFalse(obj.released) @@ -7301,8 +7290,7 @@ def test_extender_foreign_teardown_skips_native_free(self): obj = self._ExtenderResource._wrap_native_handle(0xE1) # Stamp a foreign owner: # teardown runs in a process that did not create the handle, - # so it must not free the pointer or run _release (which could touch - # native streams and deadlock after a multithreaded fork). + # so it must not free the pointer or run _release. obj._owner_pid = os.getpid() + 1 obj.close() @@ -7342,10 +7330,6 @@ def test_builder_from_archive_wraps_handle(self): self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) def test_context_build_failure_consumes_signer(self): - # c2pa_context_builder_set_signer takes ownership of the signer - # pointer immediately, so a later build failure must still leave the - # Signer consumed. - # Otherwise it holds a pointer the native side has already freed. self._use_real_frees() signer = self._make_signer() real_build = c2pa_module._lib.c2pa_context_builder_build @@ -7401,21 +7385,22 @@ def test_construction_failure_leaves_nothing_to_free(self): finally: c2pa_module._lib.c2pa_builder_from_json = real_json -def _ptr_addr(ptr): - """Address a ctypes pointer points at, or None for a null pointer. - - ctypes pointers compare by identity, not by value: two pointer objects - for the same address are unequal. Compare addresses instead. - """ - if not ptr: - return None - return ctypes.cast(ptr, ctypes.c_void_p).value - class TestManagedResourceObjects(TestContextAPIs): """Tests native resource handling management when managed manually. """ + @staticmethod + def _ptr_addr(ptr): + """Address a ctypes pointer points at, or None for a null pointer. + + ctypes pointers compare by identity, not by value: two pointer objects + for the same address are unequal. Compare addresses instead. + """ + if not ptr: + return None + return ctypes.cast(ptr, ctypes.c_void_p).value + def _instrument_frees(self): """Record frees instead of performing them, and restore on teardown. @@ -7434,9 +7419,9 @@ def _instrument_frees(self): def _free_count(self, freed, handle): """How many times `handle` was freed, ignoring unrelated frees.""" - target = _ptr_addr(handle) + target = self._ptr_addr(handle) self.assertIsNotNone(target, "cannot count frees of a null handle") - return sum(1 for ptr in freed if _ptr_addr(ptr) == target) + return sum(1 for ptr in freed if self._ptr_addr(ptr) == target) def _make_archive(self, manifest=None): archive = io.BytesIO() diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 540dfc82..e7d49b16 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -2984,8 +2984,7 @@ def make_and_drop(index): gc.collect() - # Count only this test's handles: resources dropped by other tests in - # the class can be collected at any point and land in self.freed. + # Count only this test's handles counts = {handle: count for handle, count in self._free_counts().items() if 0x30000 <= handle < 0x30000 + 200} @@ -2994,20 +2993,14 @@ def make_and_drop(index): self.assertEqual(set(counts.values()), {1}, "a dropped resource was freed more than once") + def test_settings_relayed_across_threads_stays_usable(self): + ManagedResource._free_native_ptr = self._real_free -class TestSettingsAsContextAcrossThreads(unittest.TestCase): - """Tests Settings handed between threads and reused as the basis - for Contexts, using real native handles. - """ - - def setUp(self): - self.manifest = { + manifest = { "claim_generator": "threaded_stamp_test", "format": "image/jpeg", "assertions": [], } - - def test_settings_relayed_across_threads_stays_usable(self): settings = Settings() pid = os.getpid() results = [] @@ -3016,7 +3009,7 @@ def test_settings_relayed_across_threads_stays_usable(self): def build_context_and_builder(): try: context = Context(settings=settings) - builder = Builder(self.manifest, context=context) + builder = Builder(manifest, context=context) results.append(( builder._owner_pid, context._owner_pid, builder.is_valid)) builder.close() @@ -3024,7 +3017,7 @@ def build_context_and_builder(): except Exception as exc: errors.append(exc) - # Sequential hand-off: each thread owns the Settings for its turn. + # Each thread owns the Settings for its turn. for _ in range(8): thread = threading.Thread(target=build_context_and_builder) thread.start() @@ -3040,37 +3033,6 @@ def build_context_and_builder(): self.assertTrue(valid) self.assertEqual(settings._owner_pid, pid) - def test_context_created_on_one_thread_closed_on_another(self): - created = [] - errors = [] - - def create(): - try: - created.append(Context()) - except Exception as exc: - errors.append(exc) - - def close_all(): - try: - for context in created: - context.close() - except Exception as exc: - errors.append(exc) - - maker = threading.Thread(target=create) - maker.start() - maker.join() - - closer = threading.Thread(target=close_all) - closer.start() - closer.join() - - self.assertEqual(errors, []) - self.assertEqual(len(created), 1) - for context in created: - self.assertEqual(context._lifecycle_state, LifecycleState.CLOSED) - self.assertIsNone(context._handle) - if __name__ == '__main__': unittest.main() From 2725dce369fde4902c8878f6a8e72b04d2b5ada9 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:53:21 -0700 Subject: [PATCH 18/67] fix: Baseline for new scenarios --- tests/perf/baseline.json | 343 +++++++++++++++++++++------------------ 1 file changed, 184 insertions(+), 159 deletions(-) diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index a6a75177..c151efe5 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -8,268 +8,293 @@ "arch": "aarch64" }, "reader_jpeg_legacy": { - "peak_bytes": 3830666, - "leaked_bytes": 3331494, - "total_allocations": 1365464 + "peak_bytes": 3851610, + "leaked_bytes": 3351823, + "total_allocations": 1362322 }, "reader_jpeg_with_context": { - "peak_bytes": 3824947, - "leaked_bytes": 3324219, - "total_allocations": 1354423 + "peak_bytes": 3845367, + "leaked_bytes": 3345097, + "total_allocations": 1349879 }, "reader_manifest_data_context": { - "peak_bytes": 7616236, - "leaked_bytes": 3448019, - "total_allocations": 1152354 + "peak_bytes": 7636730, + "leaked_bytes": 3468040, + "total_allocations": 1147359 }, "reader_mp4": { - "peak_bytes": 4200644, - "leaked_bytes": 3323805, - "total_allocations": 4100459 + "peak_bytes": 4222601, + "leaked_bytes": 3345724, + "total_allocations": 4095915 }, "reader_wav": { - "peak_bytes": 4501138, - "leaked_bytes": 3333747, - "total_allocations": 746935 + "peak_bytes": 4523095, + "leaked_bytes": 3355666, + "total_allocations": 742391 }, "builder_sign_jpeg_legacy": { - "peak_bytes": 8108426, - "leaked_bytes": 3485229, - "total_allocations": 1115105 + "peak_bytes": 7785129, + "leaked_bytes": 3468507, + "total_allocations": 1041412 }, "builder_sign_jpeg_with_context": { - "peak_bytes": 8108408, - "leaked_bytes": 3477526, - "total_allocations": 1103189 + "peak_bytes": 7779538, + "leaked_bytes": 3463042, + "total_allocations": 1027485 }, "builder_sign_png_legacy": { - "peak_bytes": 8002675, - "leaked_bytes": 3447638, - "total_allocations": 3887535 + "peak_bytes": 8023081, + "leaked_bytes": 3468300, + "total_allocations": 3883115 }, "builder_sign_png_with_context": { - "peak_bytes": 7994768, - "leaked_bytes": 3440151, - "total_allocations": 3875483 + "peak_bytes": 8017008, + "leaked_bytes": 3462829, + "total_allocations": 3869515 }, "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 44042681, - "leaked_bytes": 3841702, - "total_allocations": 1045285 + "peak_bytes": 45854797, + "leaked_bytes": 3840928, + "total_allocations": 1035646 }, "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 45824424, - "leaked_bytes": 3840908, - "total_allocations": 1043917 + "peak_bytes": 45844809, + "leaked_bytes": 3861014, + "total_allocations": 1037741 }, "builder_sign_png_parallel_split_pool": { - "peak_bytes": 46094336, - "leaked_bytes": 3860177, - "total_allocations": 3887276 + "peak_bytes": 46586728, + "leaked_bytes": 3868054, + "total_allocations": 3877696 }, "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 46062355, - "leaked_bytes": 3876678, - "total_allocations": 3885976 + "peak_bytes": 46082548, + "leaked_bytes": 3879161, + "total_allocations": 3879780 }, "builder_sign_gif": { - "peak_bytes": 14614794, - "leaked_bytes": 3439807, - "total_allocations": 17023655 + "peak_bytes": 14635465, + "leaked_bytes": 3461270, + "total_allocations": 17017654 }, "builder_sign_heic": { - "peak_bytes": 4677761, - "leaked_bytes": 3447623, - "total_allocations": 1569619 + "peak_bytes": 4698434, + "leaked_bytes": 3469086, + "total_allocations": 1563419 }, "builder_sign_m4a": { - "peak_bytes": 18812724, - "leaked_bytes": 3448162, - "total_allocations": 5200482 + "peak_bytes": 18833496, + "leaked_bytes": 3469085, + "total_allocations": 5194205 }, "builder_sign_webp": { - "peak_bytes": 8970587, - "leaked_bytes": 3440333, - "total_allocations": 922037 + "peak_bytes": 8991237, + "leaked_bytes": 3461271, + "total_allocations": 916145 }, "builder_sign_avi": { - "peak_bytes": 7110163, - "leaked_bytes": 3440347, - "total_allocations": 89988101 + "peak_bytes": 7130933, + "leaked_bytes": 3461270, + "total_allocations": 89982012 }, "builder_sign_mp4": { - "peak_bytes": 6224729, - "leaked_bytes": 3448147, - "total_allocations": 3794640 + "peak_bytes": 6245379, + "leaked_bytes": 3469085, + "total_allocations": 3788717 }, "builder_sign_tiff": { - "peak_bytes": 13192399, - "leaked_bytes": 3440348, - "total_allocations": 10868711 + "peak_bytes": 13213169, + "leaked_bytes": 3461271, + "total_allocations": 10862700 }, "builder_sign_jpeg_parent_of": { - "peak_bytes": 14244190, - "leaked_bytes": 3439412, - "total_allocations": 2514770 + "peak_bytes": 14265295, + "leaked_bytes": 3461665, + "total_allocations": 2506107 }, "builder_sign_jpeg_component_of": { - "peak_bytes": 14246389, - "leaked_bytes": 3440789, - "total_allocations": 2559666 + "peak_bytes": 14266996, + "leaked_bytes": 3462012, + "total_allocations": 2551180 }, "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14597099, - "leaked_bytes": 3546490, - "total_allocations": 4534820 + "peak_bytes": 14665241, + "leaked_bytes": 3614613, + "total_allocations": 4523960 }, "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14545928, - "leaked_bytes": 3440216, - "total_allocations": 5528067 + "peak_bytes": 14568780, + "leaked_bytes": 3462718, + "total_allocations": 5517180 }, "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14539155, - "leaked_bytes": 3544510, - "total_allocations": 4508197 + "peak_bytes": 14559274, + "leaked_bytes": 3564233, + "total_allocations": 4497379 }, "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14544717, - "leaked_bytes": 3441158, - "total_allocations": 5501353 + "peak_bytes": 14564839, + "leaked_bytes": 3461873, + "total_allocations": 5490592 }, "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14277459, - "leaked_bytes": 3460332, - "total_allocations": 3480521 + "peak_bytes": 14297571, + "leaked_bytes": 3481212, + "total_allocations": 3467149 }, "builder_from_archive_roundtrip": { - "peak_bytes": 14277245, - "leaked_bytes": 3460181, - "total_allocations": 3108863 + "peak_bytes": 14297349, + "leaked_bytes": 3480475, + "total_allocations": 3101030 }, "builder_with_archive_swap": { - "peak_bytes": 3660981, - "leaked_bytes": 3330239, - "total_allocations": 708348 + "peak_bytes": 3681081, + "leaked_bytes": 3350198, + "total_allocations": 704373 }, "reader_with_fragment_swap": { - "peak_bytes": 3758666, - "leaked_bytes": 3332314, - "total_allocations": 3792727 + "peak_bytes": 3778159, + "leaked_bytes": 3353205, + "total_allocations": 3787587 + }, + "with_fragment_pre_consume_rejection": { + "peak_bytes": 3778057, + "leaked_bytes": 3354795, + "total_allocations": 2094004 + }, + "with_archive_post_consume_failure": { + "peak_bytes": 3350600, + "leaked_bytes": 3308056, + "total_allocations": 175290 + }, + "with_fragment_marshalling_error": { + "peak_bytes": 3708068, + "leaked_bytes": 3352335, + "total_allocations": 2077090 + }, + "with_fragment_mixed_outcomes": { + "peak_bytes": 3779175, + "leaked_bytes": 3356294, + "total_allocations": 2656787 }, "builder_to_archive_with_ingredient": { - "peak_bytes": 14049708, - "leaked_bytes": 3318104, - "total_allocations": 1836413 + "peak_bytes": 14069232, + "leaked_bytes": 3337316, + "total_allocations": 1830896 }, "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": { - "peak_bytes": 14264719, - "leaked_bytes": 3459571, - "total_allocations": 5893306 + "peak_bytes": 14287046, + "leaked_bytes": 3481977, + "total_allocations": 5879957 }, "builder_write_ingredient_archive": { - "peak_bytes": 14049702, - "leaked_bytes": 3318102, - "total_allocations": 1810851 + "peak_bytes": 14069289, + "leaked_bytes": 3337377, + "total_allocations": 1805304 }, "builder_sign_jpeg_add_ingredient_from_archive": { - "peak_bytes": 14113307, - "leaked_bytes": 3459596, - "total_allocations": 3424465 + "peak_bytes": 14133742, + "leaked_bytes": 3480831, + "total_allocations": 3415920 }, "builder_ingredient_archive_roundtrip": { - "peak_bytes": 14264391, - "leaked_bytes": 3460351, - "total_allocations": 5146608 + "peak_bytes": 14284443, + "leaked_bytes": 3480809, + "total_allocations": 5132060 }, "builder_sign_jpeg_two_ingredient_archives": { - "peak_bytes": 14114874, - "leaked_bytes": 3461104, - "total_allocations": 4226879 + "peak_bytes": 14134560, + "leaked_bytes": 3481604, + "total_allocations": 4215728 }, "reader_error_no_manifest": { - "peak_bytes": 3544227, - "leaked_bytes": 3302551, - "total_allocations": 278514 + "peak_bytes": 3564471, + "leaked_bytes": 3323629, + "total_allocations": 276175 }, "builder_error_invalid_manifest": { - "peak_bytes": 3331458, - "leaked_bytes": 3276705, - "total_allocations": 114863 + "peak_bytes": 3352053, + "leaked_bytes": 3297079, + "total_allocations": 113926 }, "reader_string_apis": { - "peak_bytes": 3957555, - "leaked_bytes": 3324853, - "total_allocations": 2296696 + "peak_bytes": 3978113, + "leaked_bytes": 3346111, + "total_allocations": 2287335 }, "signer_construction": { - "peak_bytes": 3331349, - "leaked_bytes": 3268750, - "total_allocations": 155984 + "peak_bytes": 3350893, + "leaked_bytes": 3288137, + "total_allocations": 153245 + }, + "builder_from_context_construction": { + "peak_bytes": 3350600, + "leaked_bytes": 3288582, + "total_allocations": 112688 }, "fork_reader_collect": { - "peak_bytes": 3830817, - "leaked_bytes": 3333241, - "total_allocations": 1330058 + "peak_bytes": 3850530, + "leaked_bytes": 3353063, + "total_allocations": 1328122 }, "fork_contended_mutex": { - "peak_bytes": 7659277, - "leaked_bytes": 3461442, - "total_allocations": 67635759 + "peak_bytes": 7679019, + "leaked_bytes": 3482128, + "total_allocations": 67472694 }, "fork_thread_local_orphan": { - "peak_bytes": 3916484, - "leaked_bytes": 3419968, - "total_allocations": 1383197 + "peak_bytes": 3936170, + "leaked_bytes": 3439733, + "total_allocations": 1381055 }, "fork_gc_cycle": { - "peak_bytes": 3830375, - "leaked_bytes": 3332761, - "total_allocations": 1334044 + "peak_bytes": 3850434, + "leaked_bytes": 3353160, + "total_allocations": 1332098 }, "fork_parent_frees_after_fork": { - "peak_bytes": 5427183, - "leaked_bytes": 3329834, - "total_allocations": 24873000 + "peak_bytes": 5447584, + "leaked_bytes": 3350400, + "total_allocations": 24829257 }, "fork_child_closes_then_parent_frees": { - "peak_bytes": 5427152, - "leaked_bytes": 3329841, - "total_allocations": 24872992 + "peak_bytes": 5446620, + "leaked_bytes": 3350407, + "total_allocations": 24829254 }, "fork_child_sys_exit": { - "peak_bytes": 3831271, - "leaked_bytes": 3332312, - "total_allocations": 1338865 + "peak_bytes": 3850546, + "leaked_bytes": 3353234, + "total_allocations": 1335925 }, "fork_stream_cleanup": { - "peak_bytes": 3444268, - "leaked_bytes": 3272486, - "total_allocations": 106279 + "peak_bytes": 3464063, + "leaked_bytes": 3291969, + "total_allocations": 105340 }, "fork_swap_cleanup": { - "peak_bytes": 3661376, - "leaked_bytes": 3330987, - "total_allocations": 718355 + "peak_bytes": 3681171, + "leaked_bytes": 3350696, + "total_allocations": 714376 }, "fork_contended_mutex_swap": { - "peak_bytes": 7276901, - "leaked_bytes": 3443869, - "total_allocations": 37514894 + "peak_bytes": 7302379, + "leaked_bytes": 3475147, + "total_allocations": 35948516 }, "fork_contended_mutex_wrap": { - "peak_bytes": 7262615, - "leaked_bytes": 3474333, - "total_allocations": 35888503 + "peak_bytes": 7288748, + "leaked_bytes": 3463411, + "total_allocations": 34847186 }, "fork_consumed_signer": { - "peak_bytes": 3331615, - "leaked_bytes": 3269742, - "total_allocations": 179994 + "peak_bytes": 3350894, + "leaked_bytes": 3288906, + "total_allocations": 175055 }, "swap_chain_churn": { - "peak_bytes": 3661193, - "leaked_bytes": 3330367, - "total_allocations": 673717 + "peak_bytes": 3681161, + "leaked_bytes": 3350287, + "total_allocations": 672537 } } \ No newline at end of file From 7c946b985597012bbd6d1140880e7a754e26ca0a Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:01:26 -0700 Subject: [PATCH 19/67] fix: Rename vars for clarity --- src/c2pa/c2pa.py | 49 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index d631dbee..52c2a2f3 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1471,15 +1471,16 @@ def __init__(self): """Create new Settings with default values.""" super().__init__() - ptr = _lib.c2pa_settings_new() + settings_ptr = _lib.c2pa_settings_new() try: - _check_ffi_operation_result(ptr, "Failed to create Settings") + _check_ffi_operation_result( + settings_ptr, "Failed to create Settings") except BaseException: - if ptr: - ManagedResource._free_native_ptr(ptr) + if settings_ptr: + ManagedResource._free_native_ptr(settings_ptr) raise - self._activate(ptr) + self._activate(settings_ptr) @classmethod def from_json(cls, json_str: str) -> 'Settings': @@ -1640,11 +1641,11 @@ def __init__( if settings is None and signer is None: # Simple default context - ptr = _lib.c2pa_context_new() + context_ptr = _lib.c2pa_context_new() _check_ffi_operation_result( - ptr, "Failed to create Context" + context_ptr, "Failed to create Context" ) - self._activate(ptr) + self._activate(context_ptr) else: # Use ContextBuilder for settings/signer builder_ptr = _lib.c2pa_context_builder_new() @@ -1683,16 +1684,16 @@ def __init__( self._has_signer = True # Build consumes builder_ptr - ptr = ( + context_ptr = ( _lib.c2pa_context_builder_build(builder_ptr) ) builder_ptr = None _check_ffi_operation_result( - ptr, "Failed to build Context" + context_ptr, "Failed to build Context" ) - self._activate(ptr) + self._activate(context_ptr) except BaseException: # Free builder if build was not reached if builder_ptr is not None: @@ -2407,7 +2408,7 @@ def _create_reader(self, format_bytes, stream_obj, manifest_data: Optional manifest bytes """ if manifest_data is None: - ptr = _lib.c2pa_reader_from_stream( + reader_ptr = _lib.c2pa_reader_from_stream( format_bytes, stream_obj._stream) else: if not isinstance(manifest_data, bytes): @@ -2415,7 +2416,7 @@ def _create_reader(self, format_bytes, stream_obj, manifest_array = ( ctypes.c_ubyte * len(manifest_data)).from_buffer_copy(manifest_data) - ptr = ( + reader_ptr = ( _lib.c2pa_reader_from_manifest_data_and_stream( format_bytes, stream_obj._stream, @@ -2425,10 +2426,10 @@ def _create_reader(self, format_bytes, stream_obj, ) _check_ffi_operation_result( - ptr, + reader_ptr, Reader._ERROR_MESSAGES['reader_error'].format("Unknown error")) - self._activate(ptr) + self._activate(reader_ptr) def _init_from_file(self, path, format_bytes, manifest_data=None): @@ -2544,10 +2545,7 @@ def _init_attrs(self): self._backing_file = None # Caches for manifest JSON string and parsed data. - # These are invalidated when with_fragment() is called, because each - # new BMFF fragment can refine or update the manifest content as the - # reader progressively builds its understanding of the fragmented - # stream. They are also cleared on close() to release memory. + # These are invalidated when with_fragment() is called. self._manifest_json_str_cache = None self._manifest_data_cache = None @@ -2573,6 +2571,7 @@ def _close_streams(self): def _release(self): """Release Reader-specific resources (caches, stream, backing file). """ + self._manifest_json_str_cache = None self._manifest_data_cache = None self._close_streams() @@ -3265,13 +3264,13 @@ def __init__( if context is not None: self._init_from_context(context, json_str) else: - ptr = _lib.c2pa_builder_from_json(json_str) + builder_ptr = _lib.c2pa_builder_from_json(json_str) _check_ffi_operation_result( - ptr, + builder_ptr, Builder._ERROR_MESSAGES['builder_error'].format("Unknown error")) - self._activate(ptr) + self._activate(builder_ptr) def _init_from_context(self, context, json_str): """Initialize Builder from a ContextProvider. @@ -3296,9 +3295,9 @@ def _init_from_context(self, context, json_str): ManagedResource._free_native_ptr(builder_ptr) raise - # Adopt the handle before the consuming call: _consume_and_swap needs - # an active resource, and from here on normal cleanup owns the pointer - # whichever way the call goes. + # Adopt the handle before the consuming call: + # _consume_and_swap needs an active resource, + # and from here on normal cleanup owns the pointer. self._activate(builder_ptr) self._consume_and_swap( From f342997a176e00f8eb9bf358be3f1fa0854bc61d Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:26:45 -0700 Subject: [PATCH 20/67] fix: Refactor once more 2 --- src/c2pa/c2pa.py | 68 ++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 52c2a2f3..02447e67 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1261,7 +1261,11 @@ def _check_ffi_operation_result( Args: result: The return value from the FFI call - fallback_msg: Error message if the native library has no error details + fallback_msg: Error message if the native library has no error details. + An error message template ending in `: {}` may be passed + unformatted. An "Unknown error" fallback is filled in here, since + reaching this point means the native layer offered nothing better. + Plain messages with no placeholder are used as-is. check: Predicate that returns True when the result indicates failure. Defaults to `not r` (for pointer-returning calls). Use `lambda r: r != 0` for status-code-returning calls. @@ -1277,7 +1281,7 @@ def _check_ffi_operation_result( error = _parse_operation_result_for_error(_lib.c2pa_error()) if error: raise C2paError(error) - raise C2paError(fallback_msg) + raise C2paError(fallback_msg.format("Unknown error")) return result @@ -1780,15 +1784,6 @@ class Stream: 'stream_error': "Error cleaning up stream: {}", 'callback_error': "Error cleaning up callback {}: {}", 'cleanup_error': "Error during cleanup: {}", - 'read': "Stream is closed or not initialized during read operation", - 'memory_error': "Memory error during stream operation: {}", - 'read_error': "Error during read operation: {}", - 'seek': "Stream is closed or not initialized during seek operation", - 'seek_error': "Error during seek operation: {}", - 'write': "Stream is closed or not initialized during write operation", - 'write_error': "Error during write operation: {}", - 'flush': "Stream is closed or not initialized during flush operation", - 'flush_error': "Error during flush operation: {}" } def __init__(self, file_like_stream): @@ -2218,16 +2213,12 @@ class Reader(ManagedResource): # Class-level error messages to avoid multiple creation _ERROR_MESSAGES = { - 'unsupported': "Unsupported format", 'io_error': "IO error: {}", 'manifest_error': "Invalid manifest data: must be bytes", 'reader_error': "Failed to create reader: {}", 'cleanup_error': "Error during cleanup: {}", 'stream_error': "Error cleaning up stream: {}", - 'file_error': "Error cleaning up file: {}", - 'reader_cleanup_error': "Error cleaning up reader: {}", 'encoding_error': "Invalid UTF-8 characters in input: {}", - 'closed_error': "Reader is closed", 'fragment_error': "Failed to process fragment: {}" } @@ -2427,7 +2418,7 @@ def _create_reader(self, format_bytes, stream_obj, _check_ffi_operation_result( reader_ptr, - Reader._ERROR_MESSAGES['reader_error'].format("Unknown error")) + Reader._ERROR_MESSAGES['reader_error']) self._activate(reader_ptr) @@ -2495,7 +2486,7 @@ def _init_from_context(self, context, format_or_path, _check_ffi_operation_result(reader_ptr, Reader._ERROR_MESSAGES[ 'reader_error' - ].format("Unknown error") + ] ) except BaseException: if reader_ptr: @@ -2575,6 +2566,8 @@ def _release(self): self._manifest_json_str_cache = None self._manifest_data_cache = None self._close_streams() + # The Context is not ours to close, only to stop pinning. + self._context = None def _get_cached_manifest_data(self) -> Optional[dict]: """Get the cached manifest data, fetching and parsing if not cached. @@ -2889,12 +2882,8 @@ class Signer(ManagedResource): # Class-level error messages to avoid multiple creation _ERROR_MESSAGES = { - 'closed_error': "Signer is closed", 'cleanup_error': "Error during cleanup: {}", - 'signer_cleanup': "Error cleaning up signer: {}", 'callback_error': "Error in signer callback: {}", - 'info_error': "Error creating signer from info: {}", - 'invalid_data': "Invalid data for signing: {}", 'invalid_certs': "Invalid certificate data: {}", 'invalid_tsa': "Invalid TSA URL: {}", 'encoding_error': "Invalid UTF-8 characters in input: {}" @@ -3115,19 +3104,14 @@ class Builder(ManagedResource): _ERROR_MESSAGES = { 'builder_error': "Failed to create builder: {}", 'cleanup_error': "Error during cleanup: {}", - 'builder_cleanup': "Error cleaning up builder: {}", - 'closed_error': "Builder is closed", - 'manifest_error': "Invalid manifest data: must be string or dict", 'url_error': "Error setting remote URL: {}", + 'intent_error': "Error setting intent for Builder: {}", 'resource_error': "Error adding resource: {}", 'ingredient_error': "Error adding ingredient: {}", 'archive_read_error': "Error loading ingredient from archive: {}", 'action_error': "Error adding action: {}", 'archive_error': "Error writing archive: {}", 'archive_load_error': "Failed to load archive into builder: {}", - 'sign_error': "Error during signing: {}", - 'encoding_error': "Invalid UTF-8 characters in manifest: {}", - 'json_error': "Failed to serialize manifest JSON: {}" } @classmethod @@ -3268,7 +3252,7 @@ def __init__( _check_ffi_operation_result( builder_ptr, - Builder._ERROR_MESSAGES['builder_error'].format("Unknown error")) + Builder._ERROR_MESSAGES['builder_error']) self._activate(builder_ptr) @@ -3288,7 +3272,7 @@ def _init_from_context(self, context, json_str): _check_ffi_operation_result(builder_ptr, Builder._ERROR_MESSAGES[ 'builder_error' - ].format("Unknown error") + ] ) except BaseException: if builder_ptr: @@ -3344,7 +3328,7 @@ def set_remote_url(self, remote_url: str): _check_ffi_operation_result( result, - Builder._ERROR_MESSAGES['url_error'].format("Unknown error"), + Builder._ERROR_MESSAGES['url_error'], check=lambda r: r != 0) def set_intent( @@ -3383,7 +3367,7 @@ def set_intent( _check_ffi_operation_result( result, - "Error setting intent for Builder: Unknown error", + Builder._ERROR_MESSAGES['intent_error'], check=lambda r: r != 0) def add_resource(self, uri: str, stream: Any): @@ -3406,7 +3390,7 @@ def add_resource(self, uri: str, stream: Any): _check_ffi_operation_result( result, - Builder._ERROR_MESSAGES['resource_error'].format("Unknown error"), + Builder._ERROR_MESSAGES['resource_error'], check=lambda r: r != 0) def add_ingredient( @@ -3473,7 +3457,7 @@ def add_ingredient_from_stream( _check_ffi_operation_result( result, - Builder._ERROR_MESSAGES['ingredient_error'].format("Unknown error"), + Builder._ERROR_MESSAGES['ingredient_error'], check=lambda r: r != 0) def add_action(self, action_json: Union[str, dict]) -> None: @@ -3495,7 +3479,7 @@ def add_action(self, action_json: Union[str, dict]) -> None: _check_ffi_operation_result( result, - Builder._ERROR_MESSAGES['action_error'].format("Unknown error"), + Builder._ERROR_MESSAGES['action_error'], check=lambda r: r != 0) def to_archive(self, stream: Any) -> None: @@ -3516,7 +3500,7 @@ def to_archive(self, stream: Any) -> None: _check_ffi_operation_result( result, - Builder._ERROR_MESSAGES["archive_error"].format("Unknown error"), + Builder._ERROR_MESSAGES["archive_error"], check=lambda r: r != 0) def write_ingredient_archive(self, ingredient_id: str, stream: Any) -> None: @@ -3539,10 +3523,9 @@ def write_ingredient_archive(self, ingredient_id: str, stream: Any) -> None: result = _lib.c2pa_builder_write_ingredient_archive( self._handle, ingredient_id_str, stream_obj._stream) - _check_ffi_operation_result(result, - Builder._ERROR_MESSAGES["archive_error"].format( - "Unknown error" - ), + _check_ffi_operation_result( + result, + Builder._ERROR_MESSAGES["archive_error"], check=lambda r: r != 0) def add_ingredient_from_archive(self, stream: Any) -> None: @@ -3562,10 +3545,9 @@ def add_ingredient_from_archive(self, stream: Any) -> None: result = _lib.c2pa_builder_add_ingredient_from_archive( self._handle, stream_obj._stream) - _check_ffi_operation_result(result, - Builder._ERROR_MESSAGES["archive_read_error"].format( - "Unknown error" - ), + _check_ffi_operation_result( + result, + Builder._ERROR_MESSAGES["archive_read_error"], check=lambda r: r != 0) def with_archive(self, stream: Any) -> 'Builder': From e8504bf2fa087213c89d8b81b17c0c2a006988df Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:59:13 -0700 Subject: [PATCH 21/67] fix: Docs --- docs/native-resources-management.md | 238 +++++++++++++++++++++++----- 1 file changed, 198 insertions(+), 40 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 100936d0..21384c3b 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -2,6 +2,9 @@ `ManagedResource` is the internal base class used by the C2PA Python SDK to wrap native (Rust/FFI) pointers. When adding new wrappers around native resources `ManagedResource` should be subclassed and follow the documented lifecycle rules. +> [!NOTE] +> `ManagedResource` and the lifecycle machinery described here are internal to the SDK, not part of its public API. In most cases, code that reads and writes C2PA data should use the public wrappers (`Reader`, `Builder`, `Signer`, `Context`, `Settings`). + ## Why `ManagedResource`? `ManagedResource` is the internal base class responsible for managing native pointers owned by the C2PA Python SDK. It guarantees: @@ -85,8 +88,9 @@ Notes: | --- | --- | | **Pointer freed exactly once** | Each native pointer is passed to `c2pa_free` at most once. No leak (zero frees) and no double-free. | | **Cleanup is idempotent** | Calling `close()` (or exiting a `with` block) multiple times is safe; after the first successful cleanup, further calls do nothing. | -| **Cleanup never raises** | The cleanup path (including `_release()` and `c2pa_free`) is wrapped so that exceptions are caught and logged, never re-raised. The original exception from the `with` block (if any) is never masked. | -| **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED → ACTIVE → CLOSED. A closed resource cannot be reactivated. | +| **Cleanup never raises** | The cleanup path is wrapped so that exceptions are caught and logged, never re-raised. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. | +| **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED to ACTIVE to CLOSED. A closed resource cannot be reactivated. | +| **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_mark_consumed()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | | **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_mark_consumed()`), the object stops managing it and does not call `c2pa_free` on it. | | **Public methods validate lifecycle state** | Every public API calls `_ensure_valid_state()` before use; closed or invalid state yields `C2paError` instead of undefined behavior or crashes. | @@ -105,10 +109,10 @@ The native Rust library exposes a single C FFI function, `c2pa_free`, that deall ```python @staticmethod def _free_native_ptr(ptr): - _lib.c2pa_free(ctypes.cast(ptr, ctypes.c_void_p)) + _lib.c2pa_free(ptr) ``` -All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). The `ctypes.cast` to `c_void_p` is needed because the C function accepts a generic void pointer regardless of the original type. +All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). No explicit `ctypes.cast` is needed: `c2pa_free`'s declared argtype is `c_void_p`, so ctypes converts any pointer instance on the way in. Casting explicitly with `ctypes.cast(ptr, c_void_p)` performs the same conversion but leaves a reference cycle behind on every call, so avoid it here. `ManagedResource` guarantees that `c2pa_free` is called exactly once per pointer: not zero times (leak), not twice (double-free). @@ -120,7 +124,8 @@ Each `ManagedResource` tracks its state with a `LifecycleState` enum: stateDiagram-v2 direction LR [*] --> UNINITIALIZED : __init__() - UNINITIALIZED --> ACTIVE : native pointer created + UNINITIALIZED --> ACTIVE : _activate(handle) + ACTIVE --> ACTIVE : _swap_handle(new_handle) ACTIVE --> CLOSED : close() / __exit__ / __del__ / _mark_consumed() ``` @@ -130,7 +135,17 @@ stateDiagram-v2 The transition from ACTIVE to CLOSED is one-way. Once closed, an object cannot be reactivated. -Every public method calls `_ensure_valid_state()` before doing any work. Besides checking the lifecycle state, this method also calls `_clear_error_state()`, which resets any stale error left over from a previous native library call. Without this, an error from one operation could leak into the next one and produce a misleading error message. +Each transition has one method that performs it, and subclasses must go through them rather than assigning `_handle` or `_lifecycle_state` directly: + +| Method | Transition | What it enforces | +| --- | --- | --- | +| `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | +| `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | +| `_mark_consumed()` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side. Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | + +Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. + +Every public method calls `_ensure_valid_state()` before doing any work, which raises `C2paError` unless the resource is ACTIVE with a non-null handle. ## Ways to clean up @@ -165,10 +180,29 @@ If neither of the above is used, `__del__` attempts to free the native pointer w Cleanup must never raise an exception. A failure during cleanup (for example, the native library crashing on free) should not mask the original exception that caused the `with` block to exit. `ManagedResource` enforces this: - `close()` delegates to `_cleanup_resources()`, which wraps the entire cleanup sequence in a try/except that catches and silences all exceptions. +- `_release()` is never called directly during cleanup. It runs inside `_safe_release()`, which logs any failure with a traceback and returns normally, so a subclass whose `_release()` raises cannot stop the native pointer from being freed afterwards. - If freeing the native pointer fails, the error is logged via Python's `logging` module but not re-raised. - The state is set to `CLOSED` as the very first step, before attempting to free anything. If cleanup fails halfway, the object is still marked closed, preventing a second attempt from doing further damage. - Cleanup is idempotent. Calling `close()` on an already-closed object returns immediately. +All three cleanup entry points converge on the same method, and the exception handling sits at three different levels inside it: + +```mermaid +flowchart TD + E["close() / __exit__ / __del__"] --> CR["_cleanup_resources()"] + CR --> FP{"foreign process?"} + FP -->|yes| N["null the handle, set CLOSED,
do not free"] --> DONE([return]) + FP -->|no| ST{"already CLOSED?"} + ST -->|yes| DONE + ST -->|no| SET["set CLOSED first"] + SET --> REL["_safe_release()
logs and swallows"] + REL --> H{"handle set?"} + H -->|no| DONE + H -->|yes| FREE["_free_native_ptr()
logs on failure"] --> NULL["_handle = None"] --> DONE +``` + +The `foreign process` branch is explained under [Fork safety](#fork-safety) below. + ## Nesting resources When multiple native resources are in play at once, they can share a single `with` statement or use nested blocks. Either way, Python cleans them up in reverse order (right to left, or inner to outer). @@ -208,7 +242,7 @@ When the Reader is closed, it first releases its own resources (open file handle ## Builder lifecycle -A `Builder` follows the same pattern as Reader, with one difference: **signing consumes the builder**. The native library takes ownership of the builder's pointer during the sign operation. After signing, the builder is closed and cannot be reused. +A `Builder` follows the same pattern as Reader, with one difference: **signing closes the builder**. A Builder is single-use, so after signing it cannot be reused. ```mermaid stateDiagram-v2 @@ -219,14 +253,14 @@ stateDiagram-v2 CLOSED --> [*] note left of CLOSED - .sign() consumes the pointer - close() frees it + .sign() closes the builder + to enforce single use end note ``` -While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` consumes the native pointer (ownership transfers to the native library), so the Builder cannot be reused afterward. Closing without signing frees the pointer normally. +While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` closes the Builder when it returns, on both the success and the failure path. Closing without signing frees the pointer the same way. -After `.sign()`, the builder calls `_mark_consumed()`, which sets the handle to `None` and the state to `CLOSED`. Because the native library now owns the pointer, `ManagedResource` does not call `c2pa_free`. That would double-free memory the native library already manages. +The native sign call borrows the builder's pointer rather than taking ownership of it, so `Builder` never calls `_mark_consumed()` and the pointer is freed normally through `c2pa_free`. The close enforces single use; it is not a memory-management requirement. ## Ownership transfer @@ -234,13 +268,55 @@ Some operations transfer a native pointer from one object to another. When this `_mark_consumed()` handles this. It sets `_handle = None` and `_lifecycle_state = CLOSED` in one step. -There are two cases where this is relevant: +In the SDK this happens in one place: passing a `Signer` to a `Context`. The Context takes ownership of the Signer's native pointer, and the Signer must not be used again directly after that. + +The order of operations matters, because `c2pa_context_builder_set_signer` takes ownership on its error path as well as on success: + +```mermaid +sequenceDiagram + participant C as Caller + participant S as Signer + participant X as Context + participant N as Native lib + + C->>X: Context(settings, signer) + X->>S: _ensure_valid_state() + X->>X: copy signer._callback_cb to _signer_callback_cb + Note right of X: Pin the callback first:
the Signer is about to close + X->>N: c2pa_context_builder_set_signer(builder_ptr, handle) + + alt ctypes.ArgumentError + Note over X,N: Marshalling failed, native side never ran + X-->>C: re-raise, Signer keeps its handle + else call returned + N-->>X: result + X->>S: _mark_consumed() + Note right of S: Unconditional, before checking result:
set_signer takes ownership either way + X->>X: raise if result != 0 + end + + X->>N: c2pa_context_builder_build(builder_ptr) + N-->>X: context_ptr + X->>X: _activate(context_ptr) +``` + +Details in that sequence that are easy to get wrong: -- When a `Signer` is passed to a `Context`, the Context takes ownership of the Signer's native pointer. The Signer is marked consumed and must not be used again. +- The callback is copied to the Context *before* the transfer. `_mark_consumed()` runs `_release()`, so consuming the Signer drops its reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. +- `_mark_consumed()` runs before the result is checked, not after. A failed `set_signer` has still taken the pointer, so waiting for a successful result would leak it. +- `ctypes.ArgumentError` is the exception. It means ctypes could not marshal the arguments, so the native function never ran and never saw the pointer. The Signer still owns its handle and is left untouched. -- When `Builder.sign()` is called, the native library consumes the Builder's pointer. The Builder marks itself consumed regardless of whether the sign operation succeeds or fails, because in both cases the native library has taken the pointer. +Both `c2pa_context_builder_set_signer` and `c2pa_context_builder_build` consume what they are given, so the `builder_ptr` is freed by the error handler only when the build was never reached. -## Consume-and-return +### Adopting a handle the SDK already owns + +Ownership can also arrive from the other direction: a native call returns a pointer that needs a Python wrapper around it. `_wrap_native_handle()` is the classmethod for that. It builds an instance with `object.__new__`, runs `ManagedResource.__init__` on it (which sets the lifecycle fields and stamps the owning process ID), runs `_init_attrs()` for the subclass attribute defaults, and calls `_activate()` with the handle. + +It deliberately skips `__init__`, because `__init__` would try to create a *new* native resource. That is why attribute defaults belong in `_init_attrs()`: it is the only initialization step this path runs. + +Ownership transfers only if the call returns. If `_wrap_native_handle()` raises, no wrapper exists to free the pointer, so the caller still owns it and must free it. + +## Consume-and-swap `_mark_consumed()` closes an object permanently. A different pattern is needed when the native library must replace an object's internal state without discarding the Python-side object. This happens with fragmented media: `Reader.with_fragment()` feeds a new BMFF fragment (used in DASH/HLS streaming) into an existing Reader, and the native library must rebuild its internal representation to account for the new data. The native API does this by consuming the old pointer and returning a new one. Creating a fresh `Reader` from scratch would not work because the native library needs the accumulated state from prior fragments. @@ -260,14 +336,46 @@ stateDiagram-v2 end note ``` +The object stays `ACTIVE` throughout because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. The old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. + +### `_consume_and_swap()` + +Every call of this shape goes through one helper, which takes the FFI call as a callable and handles the outcomes: + ```python # Reader.with_fragment() internally does: -new_ptr = _lib.c2pa_reader_with_fragment(self._handle, ...) -# self._handle (old pointer) is now invalid -self._handle = new_ptr +self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_fragment(handle, format_bytes, stream), + Reader._ERROR_MESSAGES['reader_error']) ``` -The object stays `ACTIVE` throughout because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. The old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. +The call is passed as a lambda because the helper supplies the handle and, on success, replaces it via `_swap_handle()`. + +The helper exists because a null return can be ambiguous. The native function validates the borrowed pointer first, then takes ownership, then does the work, so a null result can mean either "rejected your pointer, never took it" or "took your pointer, then failed". The native error message is what tells them apart: + +| Native error | Who owns the handle | +| --- | --- | +| `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved, so the handle is kept and the resource stays `ACTIVE` | +| Any other error | Assumed taken, so `_mark_consumed()` runs and the resource goes `CLOSED`. The raised error is typed from the native message. | +| No error at all | Same ownership conclusion, but nothing to type the error from, so the caller's message is raised with `"Unknown error"` filled in. | + +This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. + +### Adopting the handle before giving it away + +`Reader._init_from_context` and `Builder._init_from_context` both create a native object, immediately `_activate()` it, and only then make the consuming call. Reduced to its shape: + +```python +self._activate(reader_ptr) + +self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_stream( + handle, format_bytes, self._own_stream._stream, + ), + Reader._ERROR_MESSAGES['reader_error']) +``` + +Activating a handle that is about to be handed to the native library looks backwards, and there are two reasons for it. `_consume_and_swap` needs an active resource to read the handle from and swap the result into. It also puts the intermediate pointer under normal cleanup before anything can go wrong with it: whichever way the consuming call goes, `close()` and `__del__` will free the pointer if the native side did not take it. The alternative, holding the pointer in a local variable across the call, means every failure path has to decide for itself whether to free it. ## Subclass-specific cleanup with `_release()` @@ -277,14 +385,56 @@ Examples from the codebase: | Class | What `_release()` cleans up | | --- | --- | -| Reader | Closes owned file handles and stream wrappers | -| Context | Drops the reference to the signer callback | +| Reader | Drops the manifest caches, closes owned file handles and stream wrappers, and drops the reference to the Context | +| Builder | Drops the reference to the Context | +| Context | Drops the reference to the signer callback. `has_signer` is left as it was: it records how the Context was configured, and stays readable after close. | | Signer | Drops the reference to the signing callback | | Settings | (no override, nothing extra to clean up) | -| Builder | (no override, nothing extra to clean up) | The cleanup order matters: `_release()` runs first (closing streams, dropping callbacks), then `c2pa_free` frees the native pointer. This order prevents the native library from accessing Python objects that no longer exist. +### Dropping a Context reference + +`Reader` and `Builder` both keep a `_context` attribute that is written once and never read. It is not dead code: it is what keeps the Context alive while the native handle depends on it. Without that reference, `Reader("image/jpeg", stream, context=Context())` would let the Context become collectable as soon as the constructor returned, even though the reader is still using it. + +Clearing it in `_release()` is the other half of that. A closed Reader has no further use for the Context, and holding the reference would keep alive an object nothing can reach through the Reader's public API. + +Dropping the reference before the native pointer is freed is safe because the native side does not depend on the Python object staying alive. `Reader::from_shared_context` clones the underlying `Arc`, so the native reader holds its own count on the context and does not care whether Python still points at it. + +## Fork safety + +`fork()` copies the calling process, including every Python object holding a native pointer. The child gets its own copy of the wrapper object, but there is still only one native allocation, and the parent owns it. + +If the child's copy were cleaned up normally, two things would go wrong. The obvious one is a double-free: the child frees a pointer the parent is still using. The subtler one is a deadlock. `fork()` only carries over the calling thread, so a native mutex held by any other thread at the moment of the fork stays locked forever in the child. Calling into the native library to free anything can block on that mutex and never return. + +So the SDK does not free native memory in a process that did not allocate it. `ManagedResource.__init__` stamps the creating process ID onto the object (`record_owner_pid`), and `is_foreign_process()` compares it against the current PID during cleanup: + +```mermaid +sequenceDiagram + participant P as Parent process + participant O as Reader object + participant F as Forked child + + P->>O: __init__ stamps _owner_pid + P->>F: fork() + Note over O,F: Child inherits a copy of the object.
One native allocation, two Python copies. + + F->>F: child's copy is cleaned up + F->>F: is_foreign_process() is true + Note right of F: Do not free: the parent owns it,
and a native mutex may be locked
by a thread that did not survive the fork + F->>F: null the handle, mark CLOSED + + P->>O: close() + P->>P: frees the native pointer normally +``` + +Both `_cleanup_resources()` and `_mark_consumed()` take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. + +The memory the child skips is not lost for good. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. + +> [!NOTE] +> `is_foreign_process()` returns `False` when no owner PID was ever recorded, so an object that somehow missed the stamp is cleaned up as before rather than leaking silently. + ## Why is `Stream` not a `ManagedResource`? `Stream` wraps a Python stream-like object (file stream or memory stream) so the native library can read from and write to it via callbacks. It does not inherit from `ManagedResource`, and it uses `c2pa_release_stream()` instead of `c2pa_free()` for cleanup. @@ -299,26 +449,32 @@ To wrap a new native resource, inherit from `ManagedResource` and follow these r ```python class NativeResource(ManagedResource): - def __init__(self, arg): - super().__init__() - - # 1. Initialize ALL instance attributes before any code - # that can raise. If __init__ fails partway through, - # __del__ will call _release(), which accesses these - # attributes. If they don't exist, _release() raises AttributeError. + def _init_attrs(self): + # 1. Declare ALL instance attributes here, not in __init__. + # _wrap_native_handle() builds instances around an existing + # handle without running __init__, and calls this instead. + # An attribute set only in __init__ would be missing there. + # This also runs before anything that can raise, so a + # half-constructed object still has what _release() reads. + super()._init_attrs() self._my_stream = None self._my_cache = None - # 2. Create the native pointer. - ptr = _lib.c2pa_my_resource_new(arg) - _check_ffi_operation_result(ptr, "Failed to create MyResource") + def __init__(self, arg): + super().__init__() + self._init_attrs() + + # 2. Create the native pointer. Pass the error message as a + # template: _check_ffi_operation_result fills in the native + # error, or "Unknown error" when there is none. + handle = _lib.c2pa_my_resource_new(arg) + _check_ffi_operation_result(handle, "Failed to create MyResource: {}") - # 3. Only set _handle and activate AFTER the FFI call - # succeeded. If it raised, _lifecycle_state stays - # UNINITIALIZED and cleanup won't try to free a - # pointer that doesn't exist. - self._handle = ptr - self._lifecycle_state = LifecycleState.ACTIVE + # 3. Take ownership only after the FFI call succeeded. + # _activate() rejects a null handle and refuses to run twice, + # so the object is never ACTIVE without a live pointer. + # Never assign self._handle or self._lifecycle_state directly. + self._activate(handle) def _release(self): # 4. Clean up class-specific resources. @@ -347,9 +503,11 @@ class NativeResource(ManagedResource): ### Troubleshooting -- If `self._my_callback = None` is set after the FFI call that can raise, and the call fails, `_release()` will try to access `self._my_callback` and crash with `AttributeError`. Always initialize attributes right after `super().__init__()`. +- If an attribute is set only in `__init__`, an instance built by `_wrap_native_handle()` will not have it, because that path never runs `__init__`. The failure shows up later as an `AttributeError` from whichever method reads the attribute, often `_release()` during cleanup. Declare attributes in `_init_attrs()` and call it from `__init__`. + +- If `_init_attrs()` is called after an FFI call that can raise, and the call fails, `_release()` will access attributes that do not exist yet and crash with `AttributeError`. Call it immediately after `super().__init__()`, before anything that can fail. -- If `_lifecycle_state = ACTIVE` is set before the FFI call and the call fails, cleanup will try to free a null or invalid pointer. Activation should happen only after a valid handle exists. +- Assigning `self._handle` or `self._lifecycle_state` directly bypasses the checks that make the lifecycle safe. `_activate()` refuses a null handle and refuses to run on an already-active object; `_swap_handle()` requires the resource to be active and the replacement non-null. Assigning the fields yourself gives up both, and the resulting bugs (an ACTIVE object with a null handle, or a silently discarded pointer) surface far from their cause. - If `_release()` raises, the exception is silently swallowed by `_cleanup_resources()`. It will not be visible unless logs are checked. Define a lifecycle for managed resources so `_release()` can check whether they need releasing. Wrap the actual release call in try/except as a fallback for unexpected failures. From f02e14b5ccb2b2587ca1355993dbe8a46927fdb4 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:45:44 -0700 Subject: [PATCH 22/67] fix: Handle handling handles (#296) * fix: Docs * fix: Error handling memory * fix: Update the docs * fix: Refactor * fix: Fix script --- docs/native-resources-management.md | 252 +++++++++++++++++++++++----- src/c2pa/c2pa.py | 87 ++++++++-- tests/perf/entrypoint.sh | 10 +- tests/test_unit_tests.py | 86 ++++++---- 4 files changed, 346 insertions(+), 89 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 100936d0..1b01362e 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -2,6 +2,9 @@ `ManagedResource` is the internal base class used by the C2PA Python SDK to wrap native (Rust/FFI) pointers. When adding new wrappers around native resources `ManagedResource` should be subclassed and follow the documented lifecycle rules. +> [!NOTE] +> `ManagedResource` and the lifecycle machinery described here are internal to the SDK. In most cases, code that reads and writes C2PA data should use the public wrappers (`Reader`, `Builder`, `Signer`, `Context`, `Settings`). + ## Why `ManagedResource`? `ManagedResource` is the internal base class responsible for managing native pointers owned by the C2PA Python SDK. It guarantees: @@ -85,8 +88,9 @@ Notes: | --- | --- | | **Pointer freed exactly once** | Each native pointer is passed to `c2pa_free` at most once. No leak (zero frees) and no double-free. | | **Cleanup is idempotent** | Calling `close()` (or exiting a `with` block) multiple times is safe; after the first successful cleanup, further calls do nothing. | -| **Cleanup never raises** | The cleanup path (including `_release()` and `c2pa_free`) is wrapped so that exceptions are caught and logged, never re-raised. The original exception from the `with` block (if any) is never masked. | -| **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED → ACTIVE → CLOSED. A closed resource cannot be reactivated. | +| **Cleanup never raises** | The cleanup path is wrapped so that exceptions are caught and logged, never re-raised. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. | +| **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED to ACTIVE to CLOSED. A closed resource cannot be reactivated. | +| **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_mark_consumed()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | | **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_mark_consumed()`), the object stops managing it and does not call `c2pa_free` on it. | | **Public methods validate lifecycle state** | Every public API calls `_ensure_valid_state()` before use; closed or invalid state yields `C2paError` instead of undefined behavior or crashes. | @@ -105,10 +109,10 @@ The native Rust library exposes a single C FFI function, `c2pa_free`, that deall ```python @staticmethod def _free_native_ptr(ptr): - _lib.c2pa_free(ctypes.cast(ptr, ctypes.c_void_p)) + _lib.c2pa_free(ptr) ``` -All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). The `ctypes.cast` to `c_void_p` is needed because the C function accepts a generic void pointer regardless of the original type. +All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). No explicit `ctypes.cast` is needed: `c2pa_free`'s declared argtype is `c_void_p`, so ctypes converts any pointer instance on the way in. Casting explicitly with `ctypes.cast(ptr, c_void_p)` performs the same conversion but leaves a reference cycle behind on every call, which creates additional load on the (Python) garbage collector. `ManagedResource` guarantees that `c2pa_free` is called exactly once per pointer: not zero times (leak), not twice (double-free). @@ -120,7 +124,8 @@ Each `ManagedResource` tracks its state with a `LifecycleState` enum: stateDiagram-v2 direction LR [*] --> UNINITIALIZED : __init__() - UNINITIALIZED --> ACTIVE : native pointer created + UNINITIALIZED --> ACTIVE : _activate(handle) + ACTIVE --> ACTIVE : _swap_handle(new_handle) ACTIVE --> CLOSED : close() / __exit__ / __del__ / _mark_consumed() ``` @@ -130,7 +135,18 @@ stateDiagram-v2 The transition from ACTIVE to CLOSED is one-way. Once closed, an object cannot be reactivated. -Every public method calls `_ensure_valid_state()` before doing any work. Besides checking the lifecycle state, this method also calls `_clear_error_state()`, which resets any stale error left over from a previous native library call. Without this, an error from one operation could leak into the next one and produce a misleading error message. +Each transition has one method that performs it, and subclasses must go through them rather than assigning `_handle` or `_lifecycle_state` directly: + +| Method | Transition | What it enforces | +| --- | --- | --- | +| `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | +| `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | +| `_mark_consumed()` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | +| `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly and closes the object, for the consume-and-swap failure path where the caller must free. Same post-state as `_mark_consumed()`; the difference is the extra `c2pa_free`. | + +Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. + +Every public method calls `_ensure_valid_state()` before doing any work, which raises `C2paError` unless the resource is ACTIVE with a non-null handle. ## Ways to clean up @@ -165,10 +181,29 @@ If neither of the above is used, `__del__` attempts to free the native pointer w Cleanup must never raise an exception. A failure during cleanup (for example, the native library crashing on free) should not mask the original exception that caused the `with` block to exit. `ManagedResource` enforces this: - `close()` delegates to `_cleanup_resources()`, which wraps the entire cleanup sequence in a try/except that catches and silences all exceptions. +- `_release()` is never called directly during cleanup. It runs inside `_safe_release()`, which logs any failure with a traceback and returns normally, so a subclass whose `_release()` raises cannot stop the native pointer from being freed afterwards. - If freeing the native pointer fails, the error is logged via Python's `logging` module but not re-raised. - The state is set to `CLOSED` as the very first step, before attempting to free anything. If cleanup fails halfway, the object is still marked closed, preventing a second attempt from doing further damage. - Cleanup is idempotent. Calling `close()` on an already-closed object returns immediately. +All three cleanup entry points converge on the same method, and the exception handling sits at three different levels inside it: + +```mermaid +flowchart TD + E["close() / __exit__ / __del__"] --> CR["_cleanup_resources()"] + CR --> FP{"foreign process?"} + FP -->|yes| N["null the handle, set CLOSED,
do not free"] --> DONE([return]) + FP -->|no| ST{"already CLOSED?"} + ST -->|yes| DONE + ST -->|no| SET["set CLOSED first"] + SET --> REL["_safe_release()
logs and swallows"] + REL --> H{"handle set?"} + H -->|no| DONE + H -->|yes| FREE["_free_native_ptr()
logs on failure"] --> NULL["_handle = None"] --> DONE +``` + +The `foreign process` branch is explained under [Fork safety](#fork-safety) below. + ## Nesting resources When multiple native resources are in play at once, they can share a single `with` statement or use nested blocks. Either way, Python cleans them up in reverse order (right to left, or inner to outer). @@ -208,7 +243,7 @@ When the Reader is closed, it first releases its own resources (open file handle ## Builder lifecycle -A `Builder` follows the same pattern as Reader, with one difference: **signing consumes the builder**. The native library takes ownership of the builder's pointer during the sign operation. After signing, the builder is closed and cannot be reused. +A `Builder` follows the same pattern as Reader, with one difference: **signing closes the builder**. A Builder is single-use, so after signing it cannot be reused. ```mermaid stateDiagram-v2 @@ -219,14 +254,14 @@ stateDiagram-v2 CLOSED --> [*] note left of CLOSED - .sign() consumes the pointer - close() frees it + .sign() closes the builder + to enforce single use end note ``` -While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` consumes the native pointer (ownership transfers to the native library), so the Builder cannot be reused afterward. Closing without signing frees the pointer normally. +While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` closes the Builder when it returns, on both the success and the failure path. Closing without signing frees the pointer the same way. -After `.sign()`, the builder calls `_mark_consumed()`, which sets the handle to `None` and the state to `CLOSED`. Because the native library now owns the pointer, `ManagedResource` does not call `c2pa_free`. That would double-free memory the native library already manages. +The native sign call borrows the builder's pointer rather than taking ownership of it, so `Builder` never calls `_mark_consumed()` and the pointer is freed normally through `c2pa_free`. The close enforces single use; it is not a memory-management requirement. ## Ownership transfer @@ -234,13 +269,55 @@ Some operations transfer a native pointer from one object to another. When this `_mark_consumed()` handles this. It sets `_handle = None` and `_lifecycle_state = CLOSED` in one step. -There are two cases where this is relevant: +In the SDK this happens in one place: passing a `Signer` to a `Context`. The Context takes ownership of the Signer's native pointer, and the Signer must not be used again directly after that. + +The order of operations matters, because `c2pa_context_builder_set_signer` takes ownership on its error path as well as on success: + +```mermaid +sequenceDiagram + participant C as Caller + participant S as Signer + participant X as Context + participant N as Native lib + + C->>X: Context(settings, signer) + X->>S: _ensure_valid_state() + X->>X: copy signer._callback_cb to _signer_callback_cb + Note right of X: Pin the callback first:
the Signer is about to close + X->>N: c2pa_context_builder_set_signer(builder_ptr, handle) + + alt ctypes.ArgumentError + Note over X,N: Marshalling failed, native side never ran + X-->>C: re-raise, Signer keeps its handle + else call returned + N-->>X: result + X->>S: _mark_consumed() + Note right of S: Unconditional, before checking result:
set_signer takes ownership either way + X->>X: raise if result != 0 + end + + X->>N: c2pa_context_builder_build(builder_ptr) + N-->>X: context_ptr + X->>X: _activate(context_ptr) +``` + +Details in that sequence that are easy to get wrong: -- When a `Signer` is passed to a `Context`, the Context takes ownership of the Signer's native pointer. The Signer is marked consumed and must not be used again. +- The callback is copied to the Context *before* the transfer. `_mark_consumed()` runs `_release()`, so consuming the Signer drops its reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. +- `_mark_consumed()` runs before the result is checked, not after. A failed `set_signer` has still taken the pointer, so waiting for a successful result would leak it. +- `ctypes.ArgumentError` is the exception. It means ctypes could not marshal the arguments, so the native function never ran and never saw the pointer. The Signer still owns its handle and is left untouched. -- When `Builder.sign()` is called, the native library consumes the Builder's pointer. The Builder marks itself consumed regardless of whether the sign operation succeeds or fails, because in both cases the native library has taken the pointer. +Both `c2pa_context_builder_set_signer` and `c2pa_context_builder_build` consume what they are given, so the `builder_ptr` is freed by the error handler only when the build was never reached. -## Consume-and-return +### Adopting a handle the SDK already owns + +Ownership can also arrive from the other direction: a native call returns a pointer that needs a Python wrapper around it. `_wrap_native_handle()` is the classmethod for that. It builds an instance with `object.__new__`, runs `ManagedResource.__init__` on it (which sets the lifecycle fields and stamps the owning process ID), runs `_init_attrs()` for the subclass attribute defaults, and calls `_activate()` with the handle. + +It deliberately skips `__init__`, because `__init__` would try to create a *new* native resource. That is why attribute defaults belong in `_init_attrs()`: it is the only initialization step this path runs. + +Ownership transfers only if the call returns. If `_wrap_native_handle()` raises, no wrapper exists to free the pointer, so the caller still owns it and must free it. + +## Consume-and-swap `_mark_consumed()` closes an object permanently. A different pattern is needed when the native library must replace an object's internal state without discarding the Python-side object. This happens with fragmented media: `Reader.with_fragment()` feeds a new BMFF fragment (used in DASH/HLS streaming) into an existing Reader, and the native library must rebuild its internal representation to account for the new data. The native API does this by consuming the old pointer and returning a new one. Creating a fresh `Reader` from scratch would not work because the native library needs the accumulated state from prior fragments. @@ -260,14 +337,57 @@ stateDiagram-v2 end note ``` +On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage below. + +### `_consume_and_swap()` + +Every call of this shape goes through one helper, which takes the FFI call as a callable and handles the outcomes: + ```python # Reader.with_fragment() internally does: -new_ptr = _lib.c2pa_reader_with_fragment(self._handle, ...) -# self._handle (old pointer) is now invalid -self._handle = new_ptr +self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_fragment(handle, format_bytes, stream), + Reader._ERROR_MESSAGES['reader_error']) ``` -The object stays `ACTIVE` throughout because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. The old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. +The call is passed as a lambda because the helper supplies the handle and, on success, replaces it via `_swap_handle()`. + +The helper exists because a null return can be ambiguous. The native function validates the borrowed pointer first, then takes ownership, then does the work, so a null result can mean either "rejected your pointer, never took it" or "took your pointer, then failed". The native error message is what tells them apart: + +| Native error | Who owns the handle | What the helper does | +| --- | --- | --- | +| `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | +| Any other error | Assumed taken, then the operation failed | `_release_handle()` frees the handle eagerly here, resource goes `CLOSED`, error typed from the native message. | +| No error at all | Same ownership conclusion, nothing to type from | `_release_handle()` frees eagerly, the caller's message is raised with `"Unknown error"` filled in. | + +Note the failure path frees eagerly (`_release_handle()`), it does not run `_mark_consumed()`. This is the dual-contract behaviour described below. + +This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. + +#### Why the failure path frees eagerly (dual contract) + +Two native contracts are in play, and the eager free is correct under both: + +- **Try-assign native (incoming):** the consuming call restores the original value to the caller on error and hands the pointer back, so the caller *must* free it. The eager `_release_handle()` is mandatory here or the handle leaks on every failed `with_fragment`/`with_archive`. +- **Released native (`c2pa-v0.90.0`):** the consuming call has already dropped the value by the time null returns, so the address is untracked. A redundant `c2pa_free` against it is a guarded no-op (returns `-1`, memory untouched), not a double-free. + +One Python path, correct under both. The native-side details (which release path drops vs. restores) are not visible from this repo — the native library is consumed as a prebuilt binary — so treat the two contracts above as the assumed native behaviour this code is written against. + +### Adopting the handle before giving it away + +`Reader._init_from_context` and `Builder._init_from_context` both create a native object, immediately `_activate()` it, and only then make the consuming call. Reduced to its shape: + +```python +self._activate(reader_ptr) + +self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_stream( + handle, format_bytes, self._own_stream._stream, + ), + Reader._ERROR_MESSAGES['reader_error']) +``` + +Activating a handle that is about to be handed to the native library looks backwards, and there are two reasons for it. `_consume_and_swap` needs an active resource to read the handle from and swap the result into. It also puts the intermediate pointer under normal cleanup before anything can go wrong with it: whichever way the consuming call goes, `close()` and `__del__` will free the pointer if the native side did not take it. The alternative, holding the pointer in a local variable across the call, means every failure path has to decide for itself whether to free it. ## Subclass-specific cleanup with `_release()` @@ -277,14 +397,56 @@ Examples from the codebase: | Class | What `_release()` cleans up | | --- | --- | -| Reader | Closes owned file handles and stream wrappers | -| Context | Drops the reference to the signer callback | +| Reader | Drops the manifest caches, closes owned file handles and stream wrappers, and drops the reference to the Context | +| Builder | Drops the reference to the Context | +| Context | Drops the reference to the signer callback. `has_signer` is left as it was: it records how the Context was configured, and stays readable after close. | | Signer | Drops the reference to the signing callback | | Settings | (no override, nothing extra to clean up) | -| Builder | (no override, nothing extra to clean up) | The cleanup order matters: `_release()` runs first (closing streams, dropping callbacks), then `c2pa_free` frees the native pointer. This order prevents the native library from accessing Python objects that no longer exist. +### Dropping a Context reference + +`Reader` and `Builder` both keep a `_context` attribute that is written once and never read. It is not dead code: it is what keeps the Context alive while the native handle depends on it. Without that reference, `Reader("image/jpeg", stream, context=Context())` would let the Context become collectable as soon as the constructor returned, even though the reader is still using it. + +Clearing it in `_release()` is the other half of that. A closed Reader has no further use for the Context, and holding the reference would keep alive an object nothing can reach through the Reader's public API. + +Dropping the reference before the native pointer is freed is safe because the native side does not depend on the Python object staying alive. `Reader::from_shared_context` clones the underlying `Arc`, so the native reader holds its own count on the context and does not care whether Python still points at it. + +## Fork safety + +`fork()` copies the calling process, including every Python object holding a native pointer. The child gets its own copy of the wrapper object, but there is still only one native allocation, and the parent owns it. + +If the child's copy were cleaned up normally, two things would go wrong. The obvious one is a double-free: the child frees a pointer the parent is still using. The subtler one is a deadlock. `fork()` only carries over the calling thread, so a native mutex held by any other thread at the moment of the fork stays locked forever in the child. Calling into the native library to free anything can block on that mutex and never return. + +So the SDK does not free native memory in a process that did not allocate it. `ManagedResource.__init__` stamps the creating process ID onto the object (`record_owner_pid`), and `is_foreign_process()` compares it against the current PID during cleanup: + +```mermaid +sequenceDiagram + participant P as Parent process + participant O as Reader object + participant F as Forked child + + P->>O: __init__ stamps _owner_pid + P->>F: fork() + Note over O,F: Child inherits a copy of the object.
One native allocation, two Python copies. + + F->>F: child's copy is cleaned up + F->>F: is_foreign_process() is true + Note right of F: Do not free: the parent owns it,
and a native mutex may be locked
by a thread that did not survive the fork + F->>F: null the handle, mark CLOSED + + P->>O: close() + P->>P: frees the native pointer normally +``` + +Both `_cleanup_resources()` and `_mark_consumed()` take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. + +The memory the child skips is not lost for good. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. + +> [!NOTE] +> `is_foreign_process()` returns `False` when no owner PID was ever recorded, so an object that somehow missed the stamp is cleaned up as before rather than leaking silently. + ## Why is `Stream` not a `ManagedResource`? `Stream` wraps a Python stream-like object (file stream or memory stream) so the native library can read from and write to it via callbacks. It does not inherit from `ManagedResource`, and it uses `c2pa_release_stream()` instead of `c2pa_free()` for cleanup. @@ -299,26 +461,32 @@ To wrap a new native resource, inherit from `ManagedResource` and follow these r ```python class NativeResource(ManagedResource): - def __init__(self, arg): - super().__init__() - - # 1. Initialize ALL instance attributes before any code - # that can raise. If __init__ fails partway through, - # __del__ will call _release(), which accesses these - # attributes. If they don't exist, _release() raises AttributeError. + def _init_attrs(self): + # 1. Declare ALL instance attributes here, not in __init__. + # _wrap_native_handle() builds instances around an existing + # handle without running __init__, and calls this instead. + # An attribute set only in __init__ would be missing there. + # This also runs before anything that can raise, so a + # half-constructed object still has what _release() reads. + super()._init_attrs() self._my_stream = None self._my_cache = None - # 2. Create the native pointer. - ptr = _lib.c2pa_my_resource_new(arg) - _check_ffi_operation_result(ptr, "Failed to create MyResource") + def __init__(self, arg): + super().__init__() + self._init_attrs() + + # 2. Create the native pointer. Pass the error message as a + # template: _check_ffi_operation_result fills in the native + # error, or "Unknown error" when there is none. + handle = _lib.c2pa_my_resource_new(arg) + _check_ffi_operation_result(handle, "Failed to create MyResource: {}") - # 3. Only set _handle and activate AFTER the FFI call - # succeeded. If it raised, _lifecycle_state stays - # UNINITIALIZED and cleanup won't try to free a - # pointer that doesn't exist. - self._handle = ptr - self._lifecycle_state = LifecycleState.ACTIVE + # 3. Take ownership only after the FFI call succeeded. + # _activate() rejects a null handle and refuses to run twice, + # so the object is never ACTIVE without a live pointer. + # Never assign self._handle or self._lifecycle_state directly. + self._activate(handle) def _release(self): # 4. Clean up class-specific resources. @@ -347,15 +515,17 @@ class NativeResource(ManagedResource): ### Troubleshooting -- If `self._my_callback = None` is set after the FFI call that can raise, and the call fails, `_release()` will try to access `self._my_callback` and crash with `AttributeError`. Always initialize attributes right after `super().__init__()`. +- If an attribute is set only in `__init__`, an instance built by `_wrap_native_handle()` will not have it, because that path never runs `__init__`. The failure shows up later as an `AttributeError` from whichever method reads the attribute, often `_release()` during cleanup. Declare attributes in `_init_attrs()` and call it from `__init__`. + +- If `_init_attrs()` is called after an FFI call that can raise, and the call fails, `_release()` will access attributes that do not exist yet and crash with `AttributeError`. Call it immediately after `super().__init__()`, before anything that can fail. -- If `_lifecycle_state = ACTIVE` is set before the FFI call and the call fails, cleanup will try to free a null or invalid pointer. Activation should happen only after a valid handle exists. +- Assigning `self._handle` or `self._lifecycle_state` directly bypasses the checks that make the lifecycle safe. `_activate()` refuses a null handle and refuses to run on an already-active object; `_swap_handle()` requires the resource to be active and the replacement non-null. Assigning the fields yourself gives up both, and the resulting bugs (an ACTIVE object with a null handle, or a silently discarded pointer) surface far from their cause. - If `_release()` raises, the exception is silently swallowed by `_cleanup_resources()`. It will not be visible unless logs are checked. Define a lifecycle for managed resources so `_release()` can check whether they need releasing. Wrap the actual release call in try/except as a fallback for unexpected failures. - `_release()` can be called more than once (via `close()` then `__del__`, or multiple `close()` calls). Make sure it handles being called on an already-cleaned-up object. Setting attributes to `None` after closing them is the standard pattern. -- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. If the pointer is freed manually and `ManagedResource` frees it again, the process crashes (double-free). +- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the consume-and-swap failure path can free eagerly without risking a double-free. Still, do not free manually — the lifecycle owns the pointer and bypassing it defeats the state checks. - If a subclass inherits from both `ManagedResource` and an ABC like `ContextProvider`, and both define a property with the same name (e.g. `is_valid`), Python resolves it using the MRO. The parent listed first in the class definition wins. If the ABC is listed first, Python finds the abstract property before the concrete one and raises `TypeError: Can't instantiate abstract class`. Always list the class with the concrete implementation first (e.g. `class Context(ManagedResource, ContextProvider)`, not `class Context(ContextProvider, ManagedResource)`). diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 02447e67..b78e06d4 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -234,9 +234,13 @@ class ManagedResource: validated, which takes ownership of it and marks the resource active. Never assign `self._handle` or `self._lifecycle_state` directly. - Call `_swap_handle(new_handle)` instead when an FFI call consumed the - current handle and returned a replacement. + current handle and returned a replacement (the success side of + `_consume_and_swap`). - Call `_mark_consumed()` when an FFI call took ownership of the handle - without returning a replacement. + without returning a replacement (e.g. `Signer` into `Context`). + - Call `_release_handle()` when a consuming FFI call fails and the caller + must free the handle: it frees eagerly, then closes like + `_mark_consumed()`. `_consume_and_swap` uses it on the failure path. - Override `_release()` to free class-specific resources (streams, caches, callbacks, etc.), called before the native pointer is freed. @@ -268,8 +272,20 @@ def _free_native_ptr(ptr): c2pa_free's argtype is c_void_p, so ctypes converts any pointer instance directly. (ctypes.cast(ptr, c_void_p) would do the same conversion but leaves a reference cycle behind on every call.) + + Returns c2pa_free's status code: + 0 when the pointer was really freed, + -1 when the pointer registry rejected an already-consumed address. + A -1 can be expected on the eager-free path when the candidate released + native memory has already dropped the value, and is gracefully handled by + the native lib too. """ - _lib.c2pa_free(ptr) + result = _lib.c2pa_free(ptr) + if result != 0: + logger.debug( + "c2pa_free returned %s for an untracked pointer ", + result) + return result def _ensure_valid_state(self): """Raise if the resource is closed or uninitialized.""" @@ -319,6 +335,38 @@ def _mark_consumed(self): self._handle = None self._lifecycle_state = LifecycleState.CLOSED + def _release_handle(self): + """Free a native handle, then close the object holding it. + Freeing synchronously at the error site leaves no window in which the + address could be reused and re-tracked. + Sets CLOSED before releasing and freeing. + """ + + if is_foreign_process(self): + self._handle = None + self._lifecycle_state = LifecycleState.CLOSED + return + + # Nothing to free, normalize states. + if self._lifecycle_state != LifecycleState.ACTIVE: + self._handle = None + self._lifecycle_state = LifecycleState.CLOSED + return + + self._lifecycle_state = LifecycleState.CLOSED + self._safe_release() + + if self._handle: + try: + ManagedResource._free_native_ptr(self._handle) + except Exception: + logger.error( + "Failed to free native %s resources", + type(self).__name__, + exc_info=True) + finally: + self._handle = None + def _activate(self, handle): """Attach a native handle to self and mark it active. Ownership of `handle` transfers here. @@ -374,14 +422,16 @@ def _swap_handle(self, new_handle): def _consume_and_swap(self, ffi_call, error_message): """Run an FFI call that consumes this handle and returns a replacement. - The native lib may take ownership partway through the call. - The native error tells if ownership was transferred: - - a pointer rejection (`_PRE_CONSUME_ERROR_TAGS`) precedes the - transfer, so the handle is still ours and is kept; - - any other error means it was taken, then the operation failed. - - The error is read without clearing it first. - The slot is sticky: c2pa_error() peeks and nothing empties it. + On success the native lib consumed the handle and returned a new one, + which we swap in. + On failure (null return, or an exception from the callback) the input + is freed eagerly and synchronously right here, then the error is raised. + The error is read before the input is freed, so the message is not lost + (in case the release steps would set a pointer tracking error). + The native error slot is sticky and thread-local: the SDK does not + clear it before this call. + The error handling therefore trusts that a failing native path set + its own error (overwriting previous one). Args: ffi_call: Callable taking the current handle, returning the @@ -395,23 +445,23 @@ def _consume_and_swap(self, ffi_call, error_message): try: new_ptr = ffi_call(self._handle) except ctypes.ArgumentError: - # Marshalling failed, so the call never reached the native side + # Marshalling failed. The call never reached the native side, # and the handle is untouched. raise except BaseException as e: - self._mark_consumed() + self._release_handle() raise C2paError(error_message.format(e)) from e if new_ptr: self._swap_handle(new_ptr) return + # Retrieve the error error = _read_native_error() if error: if any(tag in error for tag in ManagedResource._PRE_CONSUME_ERROR_TAGS): - # Rejected before ownership transferred, - # so the handle is still ours. + # Rejected before ownership transferred: the handle is still ours logger.warning( "%s: native call rejected the handle before taking " "ownership (%s); handle retained", @@ -419,11 +469,12 @@ def _consume_and_swap(self, ffi_call, error_message): error) _raise_typed_c2pa_error(error) - # Ownership transferred and then an operation failed. - self._mark_consumed() + # Ownership transferred and then the operation failed: + # free eagerly (native lib handles not double-freeing), then raise. + self._release_handle() _raise_typed_c2pa_error(error) - self._mark_consumed() + self._release_handle() raise C2paError(error_message.format("Unknown error")) @classmethod diff --git a/tests/perf/entrypoint.sh b/tests/perf/entrypoint.sh index f0f1f917..e0cbf737 100644 --- a/tests/perf/entrypoint.sh +++ b/tests/perf/entrypoint.sh @@ -16,8 +16,14 @@ else PLATFORM="x86_64-unknown-linux-gnu" fi -echo "Downloading c2pa native lib: $C2PA_VERSION / $PLATFORM" -C2PA_LIBS_PLATFORM=$PLATFORM python scripts/download_artifacts.py "$C2PA_VERSION" +# Skip the GitHub API round-trip when the lib is already on disk +# Set C2PA_FORCE_DOWNLOAD=1 to override. +if [ -z "$C2PA_FORCE_DOWNLOAD" ] && [ -f "artifacts/$PLATFORM/libc2pa_c.so" ]; then + echo "Using cached c2pa native lib: artifacts/$PLATFORM (set C2PA_FORCE_DOWNLOAD=1 to re-download)" +else + echo "Downloading c2pa native lib: $C2PA_VERSION / $PLATFORM" + C2PA_LIBS_PLATFORM=$PLATFORM python scripts/download_artifacts.py "$C2PA_VERSION" +fi # Replicate what setup.py copy_platform_libraries() does: # So the correct Linux library is here for the Dockerfile diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index f440d115..58db24d7 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -7645,41 +7645,54 @@ def test_consumed_signer_close_frees_nothing(self): "closing a consumed Signer freed a pointer the " "context now owns") - def test_builder_with_archive_null_return_consumes_self(self): + def test_builder_with_archive_null_return_frees_self(self): builder = Builder(self.test_manifest) - consumed_handle = builder._handle + released_handle = builder._handle + archive = self._make_archive() + + # Mimic an error. + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_builder_with_archive c2pa_module._lib.c2pa_builder_with_archive = lambda b, s: None + + # Instrument before the failure... + freed = self._instrument_frees() try: with self.assertRaises(Error): - builder.with_archive(self._make_archive()) + builder.with_archive(archive) finally: c2pa_module._lib.c2pa_builder_with_archive = real_call - # The FFI consumed the old handle and returned no replacement, - # so there is nothing left for this object to own... + # Nothing left to own after failing self.assertIsNone(builder._handle) self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) - freed = self._instrument_frees() + # The error path frees the old handle. + self.assertEqual(self._free_count(freed, released_handle), 1, + "error path did not free the old handle exactly once") + + # close() must not free it again. builder.close() - self.assertEqual(self._free_count(freed, consumed_handle), 0, - "close() freed a handle the FFI already consumed") + self.assertEqual(self._free_count(freed, released_handle), 1, + "close() double-freed an already-released handle") - def test_reader_with_fragment_null_return_consumes_self(self): + def test_reader_with_fragment_null_return_frees_self(self): init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") with open(init_path, "rb") as init: reader = Reader("video/mp4", init) - consumed_handle = reader._handle + released_handle = reader._handle - # The mock sets no error, which no native path does. Plant an - # operation-style one so a stale tag from another test is not read. - c2pa_module._lib.c2pa_error_set_last(b"Other: mocked null return") + # Mimic an error. + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( lambda r, f, s, frag: None) + + # Instrument before the failure so the eager free on the error path is + # counted, not just whatever close() does afterwards. + freed = self._instrument_frees() try: with open(init_path, "rb") as init, \ open(fragment_path, "rb") as frag: @@ -7691,27 +7704,34 @@ def test_reader_with_fragment_null_return_consumes_self(self): self.assertIsNone(reader._handle) self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) - freed = self._instrument_frees() - reader.close() - self.assertEqual(self._free_count(freed, consumed_handle), 0, - "close() freed a handle the FFI already consumed") + # The error path frees the old handle. + self.assertEqual(self._free_count(freed, released_handle), 1, + "error path did not free the old handle exactly once") - def test_reader_with_fragment_ffi_raise_consumes_self(self): - # If the ctypes call itself raises (not a null return), the callee has - # already consumed the old handle, so with_fragment must mark self - # consumed rather than leave a dangling pointer that close() would - # double-free. + # close() must not free it again. + reader.close() + self.assertEqual(self._free_count(freed, released_handle), 1, + "close() freed a handle the error path already freed") + + def test_reader_with_fragment_ffi_raise_frees_self(self): + # If the ctypes call itself raises, the failure runs + # through the except BaseException branch, which frees the handle. + # with_fragment must free exactly once and leave nothing for + # close() to double-free. init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") with open(init_path, "rb") as init: reader = Reader("video/mp4", init) - consumed_handle = reader._handle + released_handle = reader._handle def _raise(*_args): raise RuntimeError("boom") real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = _raise + + # Instrument before the failure so the eager free is counted. + freed = self._instrument_frees() try: with open(init_path, "rb") as init, \ open(fragment_path, "rb") as frag: @@ -7723,10 +7743,14 @@ def _raise(*_args): self.assertIsNone(reader._handle) self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) - freed = self._instrument_frees() + # The error path frees the old handle. + self.assertEqual(self._free_count(freed, released_handle), 1, + "error path did not free the old handle exactly once") + + # close() must not free it again. reader.close() - self.assertEqual(self._free_count(freed, consumed_handle), 0, - "close() freed a handle the FFI already consumed") + self.assertEqual(self._free_count(freed, released_handle), 1, + "close() freed a handle the error path already freed") # Consume-and-return ownership: the native call takes the handle partway # through its body, so a null return does not say on its own whether the @@ -7869,8 +7893,8 @@ def test_unknown_failure_drops_handle_without_freeing(self): reader = Reader("video/mp4", init) consumed_handle = reader._handle - # Not a pre-consume tag, so the mocked failure reads as unplaceable. - c2pa_module._lib.c2pa_error_set_last(b"Other: mocked null return") + # Simulate an error being set + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( lambda r, f, s, frag: None) @@ -8276,6 +8300,8 @@ def test_consumed_reader_closes_backing_file(self): backing_file = reader._backing_file self.assertFalse(backing_file.closed) + # Simulate an error being set + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( lambda r, f, s, frag: None) @@ -8296,6 +8322,8 @@ def test_consumed_builder_releases_context(self): builder = Builder(self.test_manifest, context=context) archive = self._make_archive() + # Simulate an error being set + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_builder_with_archive c2pa_module._lib.c2pa_builder_with_archive = lambda b, s: None try: @@ -8343,6 +8371,8 @@ def test_consumed_reader_clears_caches(self): reader.json() self.assertIsNotNone(reader._manifest_json_str_cache) + # Simulate an error being set + c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( lambda r, f, s, frag: None) From b0ea8ea63d269a6b02c04ed42010c3f250cc9122 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:41:03 -0700 Subject: [PATCH 23/67] fix: Refactor to simplify based on current C FFI (#297) * fix: Refactor * fix: Refactor * fix: Typo * fix: refactor * fix: Refactor * fix: Refactor 2 * fix: Clean up debug --- docs/native-resources-management.md | 19 +- src/c2pa/c2pa.py | 336 +++++++++++++--------------- tests/test_unit_tests.py | 201 +++++++++++------ 3 files changed, 291 insertions(+), 265 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 1b01362e..f5c453dc 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -142,7 +142,7 @@ Each transition has one method that performs it, and subclasses must go through | `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | | `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | | `_mark_consumed()` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | -| `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly and closes the object, for the consume-and-swap failure path where the caller must free. Same post-state as `_mark_consumed()`; the difference is the extra `c2pa_free`. | +| `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly and closes the object. Same post-state as `_mark_consumed()`. | Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. @@ -357,21 +357,14 @@ The helper exists because a null return can be ambiguous. The native function va | Native error | Who owns the handle | What the helper does | | --- | --- | --- | | `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | -| Any other error | Assumed taken, then the operation failed | `_release_handle()` frees the handle eagerly here, resource goes `CLOSED`, error typed from the native message. | -| No error at all | Same ownership conclusion, nothing to type from | `_release_handle()` frees eagerly, the caller's message is raised with `"Unknown error"` filled in. | - -Note the failure path frees eagerly (`_release_handle()`), it does not run `_mark_consumed()`. This is the dual-contract behaviour described below. +| Any other error | Taken, then the operation failed | `_mark_consumed()`: the native side already dropped the value, so nothing is freed here; resource goes `CLOSED`, error typed from the native message. | +| No error at all | Unknown (no released path reaches here) | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. -#### Why the failure path frees eagerly (dual contract) - -Two native contracts are in play, and the eager free is correct under both: - -- **Try-assign native (incoming):** the consuming call restores the original value to the caller on error and hands the pointer back, so the caller *must* free it. The eager `_release_handle()` is mandatory here or the handle leaks on every failed `with_fragment`/`with_archive`. -- **Released native (`c2pa-v0.90.0`):** the consuming call has already dropped the value by the time null returns, so the address is untracked. A redundant `c2pa_free` against it is a guarded no-op (returns `-1`, memory untouched), not a double-free. +#### Why a non-tag error does not free -One Python path, correct under both. The native-side details (which release path drops vs. restores) are not visible from this repo — the native library is consumed as a prebuilt binary — so treat the two contracts above as the assumed native behaviour this code is written against. +The binding targets one native contract, the released C FFI (`c2pa-v0.90.0`). Its consuming calls reject a borrowed pointer up front with a `_PRE_CONSUME_ERROR_TAGS` tag (handle retained), or take ownership and, on any later failure, drop the value themselves. So a non-tag error means the value is already gone: `_mark_consumed()` is exact, and a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address in another thread. `_release_handle()` stays only where ownership is genuinely unknown — an async exception mid-call, or the no-error fallthrough no released path produces — where the guarded free is the right default (a real free if the handle is ours, a `-1` no-op if not). The native-side details are not visible from this repo (the library is a prebuilt binary), so treat the contract above as the assumed native behaviour this code is written against. ### Adopting the handle before giving it away @@ -525,7 +518,7 @@ class NativeResource(ManagedResource): - `_release()` can be called more than once (via `close()` then `__del__`, or multiple `close()` calls). Make sure it handles being called on an already-cleaned-up object. Setting attributes to `None` after closing them is the standard pattern. -- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the consume-and-swap failure path can free eagerly without risking a double-free. Still, do not free manually — the lifecycle owns the pointer and bypassing it defeats the state checks. +- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the unknown-ownership failure paths can free eagerly without risking a double-free. Still, do not free manually — the lifecycle owns the pointer and bypassing it defeats the state checks. - If a subclass inherits from both `ManagedResource` and an ABC like `ContextProvider`, and both define a property with the same name (e.g. `is_valid`), Python resolves it using the MRO. The parent listed first in the class definition wins. If the ABC is listed first, Python finds the abstract property before the concrete one and raises `TypeError: Can't instantiate abstract class`. Always list the class with the concrete implementation first (e.g. `class Context(ManagedResource, ContextProvider)`, not `class Context(ContextProvider, ManagedResource)`). diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index b78e06d4..04e65af9 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -236,11 +236,12 @@ class ManagedResource: - Call `_swap_handle(new_handle)` instead when an FFI call consumed the current handle and returned a replacement (the success side of `_consume_and_swap`). - - Call `_mark_consumed()` when an FFI call took ownership of the handle - without returning a replacement (e.g. `Signer` into `Context`). - - Call `_release_handle()` when a consuming FFI call fails and the caller - must free the handle: it frees eagerly, then closes like - `_mark_consumed()`. `_consume_and_swap` uses it on the failure path. + - Call `_teardown(free_handle=False)` when an FFI call took ownership of + the handle without returning a replacement: the new owner frees it, + so this does not. + - Call `_release_handle()` when a consuming FFI call fails with ownership + unknown: it frees eagerly (guarded), then closes. `_consume_and_swap` + uses it on that failure path. - Override `_release()` to free class-specific resources (streams, caches, callbacks, etc.), called before the native pointer is freed. @@ -306,7 +307,8 @@ def _release(self): """ def _safe_release(self): - """Run _release(), logging (never raising) if it fails. + """Run _release(), logging and swallowing ordinary errors so teardown + continues. Interrupts (KeyboardInterrupt, SystemExit) propagate. """ try: self._release() @@ -317,55 +319,44 @@ def _safe_release(self): exc_info=True, ) - def _mark_consumed(self): - """Mark as consumed by an FFI call that took ownership - of native resources e.g. pointers. This means we should not - call clean-up here anymore, and leave it to the new owner. - """ + def _teardown(self, free_handle: bool): + """Close the object: run _release, optionally free the handle, null it. + The one place that owns the foreign-process rule, the CLOSED ordering + and the guarded free. free_handle=False (consumed) frees nothing; the + new owner does. + """ if is_foreign_process(self): self._handle = None self._lifecycle_state = LifecycleState.CLOSED return - # Callers raise straight after consuming, so a failing _release() - # here would mask the error they are reporting. - self._safe_release() - - self._handle = None self._lifecycle_state = LifecycleState.CLOSED + try: + self._safe_release() + finally: + # Free in finally so an interrupt escaping _release() (a BaseException + # _safe_release does not swallow) still frees an owned handle. State + # is already CLOSED, so no later teardown path would retry it. + handle, self._handle = self._handle, None + if free_handle and handle: + try: + ManagedResource._free_native_ptr(handle) + except Exception: + logger.error( + "Failed to free native %s resources", + type(self).__name__, + exc_info=True) def _release_handle(self): - """Free a native handle, then close the object holding it. - Freeing synchronously at the error site leaves no window in which the - address could be reused and re-tracked. - Sets CLOSED before releasing and freeing. + """Free this handle, then close the object. Used only where ownership is + unknown (a guarded free is a real free if ours, a no-op if not). """ - - if is_foreign_process(self): - self._handle = None - self._lifecycle_state = LifecycleState.CLOSED - return - - # Nothing to free, normalize states. if self._lifecycle_state != LifecycleState.ACTIVE: self._handle = None self._lifecycle_state = LifecycleState.CLOSED return - - self._lifecycle_state = LifecycleState.CLOSED - self._safe_release() - - if self._handle: - try: - ManagedResource._free_native_ptr(self._handle) - except Exception: - logger.error( - "Failed to free native %s resources", - type(self).__name__, - exc_info=True) - finally: - self._handle = None + self._teardown(free_handle=True) def _activate(self, handle): """Attach a native handle to self and mark it active. @@ -419,49 +410,58 @@ def _swap_handle(self, new_handle): # so it is still ours to deal with. _PRE_CONSUME_ERROR_TAGS = ("UntrackedPointer:", "WrongPointerType:") - def _consume_and_swap(self, ffi_call, error_message): - """Run an FFI call that consumes this handle and returns a replacement. + def _invoke_consume(self, ffi_call, error_message): + """Run an FFI call that consumes this handle, returning its raw result. - On success the native lib consumed the handle and returned a new one, - which we swap in. - On failure (null return, or an exception from the callback) the input - is freed eagerly and synchronously right here, then the error is raised. - The error is read before the input is freed, so the message is not lost - (in case the release steps would set a pointer tracking error). - The native error slot is sticky and thread-local: the SDK does not - clear it before this call. - The error handling therefore trusts that a failing native path set - its own error (overwriting previous one). + A marshalling ArgumentError is re-raised untouched: the call never + reached the native side, so the handle is still ours and unchanged. An + ordinary Exception from the call frees the handle before raising; an + interrupt (KeyboardInterrupt/SystemExit) propagates untouched and the + still-ACTIVE handle is freed later at close()/GC (a guarded free). + + The caller inspects the returned result to tell success from failure + (the convention differs per call) and routes a failure to + _raise_consume_failure. Args: - ffi_call: Callable taking the current handle, returning the - replacement or null. - error_message: Format string with one placeholder, used when the - native layer offers no error of its own. + ffi_call: Callable taking the current handle, returning the native + result (a replacement pointer, a status code, ...). + error_message: Format string with one placeholder, used to wrap a + callback exception. Raises: - C2paError: If the call fails, typed by native error when one. + ctypes.ArgumentError: If marshalling failed; handle untouched. + C2paError: If the call raised any other exception. """ try: - new_ptr = ffi_call(self._handle) + return ffi_call(self._handle) except ctypes.ArgumentError: - # Marshalling failed. The call never reached the native side, - # and the handle is untouched. + # Marshalling failed: the call never reached native, so the handle + # is untouched and still ours. Re-raise as-is. raise - except BaseException as e: + except Exception as e: self._release_handle() raise C2paError(error_message.format(e)) from e - if new_ptr: - self._swap_handle(new_ptr) - return + def _raise_consume_failure(self, error_message): + """Raise the error from an FFI handler consuming call. + + The native error is read before any free so a free's own + pointer-tracking error cannot overwrite it: the native error slot is + sticky and thread-local and the SDK does not clear it before the call, + so this trusts that the failing native path set its own error. - # Retrieve the error + Args: + error_message: Format string with one placeholder, used when the + native layer offers no error of its own. + + Raises: + C2paError: Always; typed by the native error when there is one. + """ error = _read_native_error() if error: if any(tag in error for tag in ManagedResource._PRE_CONSUME_ERROR_TAGS): - # Rejected before ownership transferred: the handle is still ours logger.warning( "%s: native call rejected the handle before taking " "ownership (%s); handle retained", @@ -469,14 +469,52 @@ def _consume_and_swap(self, ffi_call, error_message): error) _raise_typed_c2pa_error(error) - # Ownership transferred and then the operation failed: - # free eagerly (native lib handles not double-freeing), then raise. - self._release_handle() + # A non-tag error means the native side took ownership then failed, + # dropping the value itself: mark consumed, do not free (a free here + # would be a guarded no-op that dirties the error slot and races a + # recycled address in other threads). + self._teardown(free_handle=False) _raise_typed_c2pa_error(error) + # No error in the slot: ownership is unknown, so free defensively. self._release_handle() raise C2paError(error_message.format("Unknown error")) + def _consume_and_swap(self, ffi_call, error_message): + """Run an FFI call that consumes this handle and returns a replacement. + On success the native lib consumed the handle and returned a new one, + which we swap in. A null return is a failure. + """ + new_ptr = self._invoke_consume(ffi_call, error_message) + if new_ptr: + self._swap_handle(new_ptr) + return + self._raise_consume_failure(error_message) + + def _consume_no_replacement(self, ffi_call, error_message): + """Run an FFI call that consumes this handle on success, when the native + call returns a status code (0 = success) rather than a replacement + handle. A non-zero status is a failure routed to + _raise_consume_failure. + """ + result = self._invoke_consume(ffi_call, error_message) + if result == 0: + self._teardown(free_handle=False) + return + self._raise_consume_failure(error_message) + + def _consume_into(self, ffi_call, error_message): + """Run an FFI call that consumes this handle and returns a *different* + object's pointer. On success this handle is consumed (mark, don't free) + and the new pointer is returned for the caller to own. A null return is + a failure routed to _raise_consume_failure. + """ + result = self._invoke_consume(ffi_call, error_message) + if result: + self._teardown(free_handle=False) + return result + self._raise_consume_failure(error_message) + @classmethod def _wrap_native_handle(cls, handle): """Build a brand-new instance around an already-valid, @@ -520,18 +558,7 @@ def _cleanup_resources(self): hasattr(self, '_lifecycle_state') and self._lifecycle_state != LifecycleState.CLOSED ): - self._lifecycle_state = LifecycleState.CLOSED - self._safe_release() - if hasattr(self, '_handle') and self._handle: - try: - ManagedResource._free_native_ptr(self._handle) - except Exception: - logger.error( - "Failed to free native %s resources", - type(self).__name__, - ) - finally: - self._handle = None + self._teardown(free_handle=True) except Exception: pass @@ -1268,41 +1295,6 @@ def _raise_typed_c2pa_error(error_str: str) -> None: raise C2paError(error_str) -def _parse_operation_result_for_error( - result: ctypes.c_void_p | None, - check_error: bool = True) -> Optional[str]: - """Helper function to handle string results from C2PA functions. - - When result is falsy and check_error is True, this function retrieves the - error from the native library, parses it, and raises a typed C2paError. - - When result is truthy (a pointer to an error string), this function - converts it to a Python string, parses it, and raises a typed C2paError. - - Args: - result: A pointer to a result string, or None/falsy on error - check_error: Whether to check for errors when result is falsy - - Returns: - None if no error occurred - - Raises: - C2paError subclass: The appropriate typed exception if an error occurred - """ - if not result: # pragma: no cover - if check_error: - error_str = _read_native_error() - if error_str: - _raise_typed_c2pa_error(error_str) - return None - - # In the case result would be a string already (error message) - error_str = _convert_to_py_string(result) - if error_str: - _raise_typed_c2pa_error(error_str) - return None - - def _check_ffi_operation_result( result, fallback_msg, @@ -1329,9 +1321,9 @@ def _check_ffi_operation_result( C2paError: If the check indicates failure """ if check(result): - error = _parse_operation_result_for_error(_lib.c2pa_error()) + error = _read_native_error() if error: - raise C2paError(error) + _raise_typed_c2pa_error(error) raise C2paError(fallback_msg.format("Unknown error")) return result @@ -1530,7 +1522,7 @@ def __init__(self): try: _check_ffi_operation_result( settings_ptr, "Failed to create Settings") - except BaseException: + except Exception: if settings_ptr: ManagedResource._free_native_ptr(settings_ptr) raise @@ -1578,11 +1570,11 @@ def set(self, path: str, value: str) -> 'Settings': path_bytes = _to_utf8_bytes(path, "settings path") value_bytes = _to_utf8_bytes(value, "settings value") - result = _lib.c2pa_settings_set_value( - self._handle, path_bytes, value_bytes - ) - if result != 0: - _parse_operation_result_for_error(None) + _check_ffi_operation_result( + _lib.c2pa_settings_set_value( + self._handle, path_bytes, value_bytes), + "Failed to set settings value", + check=lambda r: r != 0) return self @@ -1603,11 +1595,11 @@ def update( data_bytes = _to_utf8_bytes(data, "settings data") - result = _lib.c2pa_settings_update_from_string( - self._handle, data_bytes, b"json" - ) - if result != 0: - _parse_operation_result_for_error(None) + _check_ffi_operation_result( + _lib.c2pa_settings_update_from_string( + self._handle, data_bytes, b"json"), + "Failed to update settings", + check=lambda r: r != 0) return self @@ -1675,6 +1667,18 @@ class Context(ManagedResource, ContextProvider): used directly again after that. """ + class _NativeBuilder(ManagedResource): + """Short-lived wrapper so the native context builder rides the normal + lifecycle: any failure inside its `with` block frees it via close() + unless a consuming call already took it. + """ + + def __init__(self): + super().__init__() + ptr = _lib.c2pa_context_builder_new() + _check_ffi_operation_result(ptr, "Failed to create ContextBuilder") + self._activate(ptr) + def __init__( self, settings: Optional['Settings'] = None, @@ -1702,61 +1706,31 @@ def __init__( ) self._activate(context_ptr) else: - # Use ContextBuilder for settings/signer - builder_ptr = _lib.c2pa_context_builder_new() - _check_ffi_operation_result( - builder_ptr, "Failed to create ContextBuilder" - ) - - try: + # Any failure inside the with frees the builder via close(); + # a successful build consumes it, so close() is then a no-op. + with self._NativeBuilder() as nb: if settings is not None: - result = ( + _check_ffi_operation_result( _lib.c2pa_context_builder_set_settings( - builder_ptr, settings._c_settings, - ) - ) - if result != 0: - _parse_operation_result_for_error(None) + nb._handle, settings._c_settings), + "Failed to set settings on Context", + check=lambda r: r != 0) if signer is not None: signer._ensure_valid_state() - # c2pa_context_builder_set_signer takes ownership of the - # signer pointer , on its error path as well as on success. + # A rejected signer is retained, not closed and leaked. self._signer_callback_cb = signer._callback_cb - try: - result = ( - _lib.c2pa_context_builder_set_signer( - builder_ptr, signer._handle, - ) - ) - except ctypes.ArgumentError: - # Marshalling failed, so the call never reached the - # native side and the signer was never taken. - raise - signer._mark_consumed() - if result != 0: - _parse_operation_result_for_error(None) + signer._consume_no_replacement( + lambda h: _lib.c2pa_context_builder_set_signer( + nb._handle, h), + "Failed to set signer on Context: {}") self._has_signer = True - # Build consumes builder_ptr - context_ptr = ( - _lib.c2pa_context_builder_build(builder_ptr) - ) - builder_ptr = None - - _check_ffi_operation_result( - context_ptr, "Failed to build Context" - ) + context_ptr = nb._consume_into( + lambda h: _lib.c2pa_context_builder_build(h), + "Failed to build Context: {}") - self._activate(context_ptr) - except BaseException: - # Free builder if build was not reached - if builder_ptr is not None: - try: - ManagedResource._free_native_ptr(builder_ptr) - except Exception: - pass - raise + self._activate(context_ptr) def _init_attrs(self): super()._init_attrs() @@ -2539,7 +2513,7 @@ def _init_from_context(self, context, format_or_path, 'reader_error' ] ) - except BaseException: + except Exception: if reader_ptr: ManagedResource._free_native_ptr(reader_ptr) raise @@ -2575,7 +2549,7 @@ def _init_from_context(self, context, format_or_path, self._own_stream._stream, ), Reader._ERROR_MESSAGES['reader_error']) - except BaseException: + except Exception: self._close_streams() raise @@ -3248,7 +3222,7 @@ def from_archive( try: # A builder from an archive here carries no context. return cls._wrap_native_handle(handle) - except BaseException: + except Exception: # No instance took ownership, so the handle is still ours. ManagedResource._free_native_ptr(handle) raise @@ -3325,7 +3299,7 @@ def _init_from_context(self, context, json_str): 'builder_error' ] ) - except BaseException: + except Exception: if builder_ptr: ManagedResource._free_native_ptr(builder_ptr) raise @@ -3685,7 +3659,7 @@ def _sign_internal( # Closing here ensures resources clean up, # and single use/single sign done by a Builder. self.close() - except BaseException as e: + except Exception as e: self.close() raise C2paError(f"Error during signing: {e}") from e diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 58db24d7..4cc05aa1 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -6934,9 +6934,10 @@ class TestManagedResourceLifecycle(unittest.TestCase): the _owner_pid stamp that governs which process may free a handle, and the ownership hand-offs between Python and the native library. - setUp records frees instead of performing them, so a miscount reads as a - leak or a double-free rather than a crash. - Tests holding real handles call _use_real_frees() first. + For testing: setUp records frees instead of performing them, + so a miscount reads as memory handling issue. + Tests releasing real handles call _use_real_frees() first to + restore release behavior. """ class _FakeHandleResource(ManagedResource): @@ -6961,10 +6962,10 @@ def _release(self): self.release_calls += 1 class _ExtenderResource(ManagedResource): - """Am extender that owns a raw handle and wraps it via + """For testing: An extender that owns a raw handle and wraps it via _wrap_native_handle. It carries several attributes of its own, all defaulted in _init_attrs (not __init__), and _release reads them, so a - missing attribute would surface as an AttributeError on teardown. + missing attribute would surface as an AttributeError on test teardown. """ def _init_attrs(self): @@ -7080,8 +7081,7 @@ def test_swap_handle_does_not_free_consumed_handle(self): res._swap_handle(0xAAA2) - # The FFI already owns and frees the old pointer, - # so freeing it here would be a double-free. + # The FFI already owns and frees the old pointer. self.assertEqual(self.freed, []) self.assertEqual(res._handle, 0xAAA2) @@ -7188,7 +7188,7 @@ def test_foreign_child_skips_free_for_wrapped_and_swapped(self): self.assertEqual(swapped._lifecycle_state, LifecycleState.CLOSED) self.assertIsNone(swapped._handle) - # A second foreign teardown is a no-op: still nothing freed. + # A second foreign teardown is a no-op. wrapped.close() swapped.close() self.assertEqual(self.freed, []) @@ -7203,8 +7203,8 @@ def test_owning_process_frees_wrapped_and_swapped_exactly_once(self): swapped._swap_handle(0xC6) swapped.close() - # 0xC5 was consumed by the (simulated) FFI swap, - # so only the replacement is ours to free. + # 0xC5 was consumed by the test FFI swap. + # Only the replacement must be freed here. self.assertEqual(self._free_counts(), {0xC4: 1, 0xC6: 1}) def test_foreign_child_skips_release(self): @@ -7222,46 +7222,46 @@ def test_foreign_child_skips_release(self): def test_consumed_resource_frees_nothing_in_either_process(self): owned = self._FakeHandleResource() owned._activate(0xE1) - owned._mark_consumed() + owned._teardown(free_handle=False) owned.close() foreign = self._FakeHandleResource() foreign._activate(0xE2) - foreign._mark_consumed() + foreign._teardown(free_handle=False) foreign._owner_pid = os.getpid() + 1 foreign.close() self.assertEqual(self.freed, []) - # Consuming a handle hands the native pointer to a new owner, - # but the Python-side resources are still ours and we need to free. - def test_mark_consumed_releases_python_resources(self): + # Consuming a handle hands the native pointer to a new owner. + # The Python-side resources are still ours to free. + def test_teardown_consumed_releases_python_resources(self): res = self._ReleaseRecordingResource() res._activate(0xF1) - res._mark_consumed() + res._teardown(free_handle=False) self.assertEqual(res.release_calls, 1) self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) self.assertIsNone(res._handle) self.assertEqual(self.freed, []) - def test_mark_consumed_swallows_failing_release(self): + def test_teardown_consumed_swallows_failing_release(self): res = self._CallbackHoldingResource() res._activate(0xF2) with self.assertLogs("c2pa", level="ERROR"): - res._mark_consumed() + res._teardown(free_handle=False) self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) self.assertIsNone(res._handle) - def test_mark_consumed_in_foreign_process_skips_release(self): + def test_teardown_consumed_in_foreign_process_skips_release(self): res = self._ReleaseRecordingResource() res._activate(0xF3) res._owner_pid = os.getpid() + 1 - res._mark_consumed() + res._teardown(free_handle=False) self.assertEqual(res.release_calls, 0) self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) @@ -7278,7 +7278,7 @@ def test_extender_wraps_handle_fully_built(self): self.assertTrue(obj.is_valid) self.assertEqual(obj._owner_pid, os.getpid()) - # _release reads those attributes, so a missing one would raise here. + # _release reads those attributes, so a missing one will raise here. obj.close() obj.close() @@ -7298,14 +7298,14 @@ def test_extender_foreign_teardown_skips_native_free(self): self.assertEqual(self.freed, [], "forked child freed a handle its parent still owns") self.assertFalse(obj.released, "foreign teardown ran _release") - # The child marks its own copy closed and nulls the handle: safe (the - # parent holds a separate copy) and it stops the child reusing a - # parent-owned handle. + # The child marks its own copy closed and nulls the handle: + # the parent holds a separate copy and it stops the + # child reusing a parent-owned handle. self.assertEqual(obj._lifecycle_state, LifecycleState.CLOSED) self.assertIsNone(obj._handle) - # A second foreign teardown stays a no-op, and any operation on the - # now-closed child copy fails loudly instead of reaching native code. + # A second foreign teardown stays a no-op, + # and any operation on the now-closed child copy fails. obj.close() self.assertEqual(self.freed, []) with self.assertRaises(Error): @@ -7368,7 +7368,7 @@ def test_context_with_signer_consumes_it_on_success(self): def test_construction_failure_leaves_nothing_to_free(self): # Activation happens after the null check, so a failed construction - # leaves no handle on the object for __del__ to find. + # has no handle on the object that __del__ can find. real_new = c2pa_module._lib.c2pa_context_new c2pa_module._lib.c2pa_context_new = lambda: None try: @@ -7385,6 +7385,71 @@ def test_construction_failure_leaves_nothing_to_free(self): finally: c2pa_module._lib.c2pa_builder_from_json = real_json + def test_context_build_null_return_frees_builder(self): + # Set a pre-consume tag in the error slot to mock a pointer rejection. + settings = Settings() + c2pa_module._lib.c2pa_error_set_last( + b"UntrackedPointer: mocked pre-consume rejection") + real_build = c2pa_module._lib.c2pa_context_builder_build + c2pa_module._lib.c2pa_context_builder_build = lambda ptr: None + try: + with self.assertRaises(Error): + Context(settings=settings) + finally: + c2pa_module._lib.c2pa_context_builder_build = real_build + + # One free: the un-consumed builder. + # Settings borrows, so it is not freed here. + self.assertEqual(len(self.freed), 1, + "un-consumed builder leaked on build failure") + settings.close() + + def test_consume_no_replacement_marks_consumed_on_success(self): + res = self._FakeHandleResource() + res._activate(0xCAFE) + + res._consume_no_replacement(lambda h: 0, "set failed: {}") + + # Native took ownership. + self.assertEqual(self.freed, []) + self.assertIsNone(res._handle) + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + + def test_consume_no_replacement_retains_on_pre_consume_tag(self): + res = self._FakeHandleResource() + res._activate(0xCAFE) + real_read = c2pa_module._read_native_error + c2pa_module._read_native_error = lambda: "UntrackedPointer: rejected" + try: + with self.assertRaises(Error): + res._consume_no_replacement(lambda h: -1, "set failed: {}") + finally: + c2pa_module._read_native_error = real_read + + # Rejected before ownership transferred: handle retained. + self.assertEqual(res._handle, 0xCAFE) + self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) + self.assertEqual(self.freed, []) + res.close() + self.assertEqual(self.freed, [0xCAFE]) + + def test_consume_no_replacement_marks_consumed_on_other_error(self): + res = self._FakeHandleResource() + res._activate(0xCAFE) + real_read = c2pa_module._read_native_error + c2pa_module._read_native_error = lambda: "OtherError: boom" + try: + with self.assertRaises(Error): + res._consume_no_replacement(lambda h: -1, "set failed: {}") + finally: + c2pa_module._read_native_error = real_read + + # A non-tag error means native took ownership then failed and dropped + # the value itself: mark consumed, do not free. + self.assertEqual(self.freed, []) + self.assertIsNone(res._handle) + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + class TestManagedResourceObjects(TestContextAPIs): """Tests native resource handling management when managed manually. @@ -7403,11 +7468,6 @@ def _ptr_addr(ptr): def _instrument_frees(self): """Record frees instead of performing them, and restore on teardown. - - This patches the base class, so every ManagedResource freed while the - patch is installed lands in the list, including objects the garbage - collector reclaims mid-test. Ask _free_count() about one handle rather - than asserting on the length of the list. """ freed = [] real_free = ManagedResource._free_native_ptr @@ -7545,7 +7605,7 @@ def test_builder_with_archive_swaps_the_handle(self): self.assertNotEqual(builder._handle, original_handle, "the native handle was not replaced") self.assertEqual(builder._lifecycle_state, LifecycleState.ACTIVE) - # The replacement came from this process, so the stamp still applies. + # The replacement came from this process, the stamp still applies. self.assertEqual(builder._owner_pid, original_stamp) self.assertEqual(builder._owner_pid, os.getpid()) builder.close() @@ -7589,8 +7649,7 @@ def test_swapped_builder_is_freed_exactly_once(self): builder.close() builder.close() - # Only the replacement is ours to free: the original was consumed by - # the FFI call that returned it. + # Only the replacement is must be freed here. self.assertEqual(self._free_count(freed, swapped_handle), 1) self.assertEqual(self._free_count(freed, original_handle), 0) @@ -7632,8 +7691,8 @@ def test_context_consumes_signer_but_not_settings(self): def test_consumed_signer_close_frees_nothing(self): signer = self._ctx_make_signer() - # Captured before the context consumes it: close() nulls the handle, - # so afterwards there is no pointer left to identify the free by. + # Captured before the context consumes it: + # close() nulls the handle, there is no pointer left to identify the free by. signer_handle = signer._handle context = Context(signer=signer) self.addCleanup(context.close) @@ -7645,12 +7704,13 @@ def test_consumed_signer_close_frees_nothing(self): "closing a consumed Signer freed a pointer the " "context now owns") - def test_builder_with_archive_null_return_frees_self(self): + def test_builder_with_archive_null_return_marks_consumed(self): builder = Builder(self.test_manifest) released_handle = builder._handle archive = self._make_archive() - # Mimic an error. + # Mimic a non-tag error: native took ownership then failed and dropped + # the value itself, so the handle is marked consumed, not freed. c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_builder_with_archive c2pa_module._lib.c2pa_builder_with_archive = lambda b, s: None @@ -7663,35 +7723,37 @@ def test_builder_with_archive_null_return_frees_self(self): finally: c2pa_module._lib.c2pa_builder_with_archive = real_call - # Nothing left to own after failing + # Nothing left to own after failing. self.assertIsNone(builder._handle) self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) - # The error path frees the old handle. - self.assertEqual(self._free_count(freed, released_handle), 1, - "error path did not free the old handle exactly once") + # A non-tag error marks consumed: no free (a free here would be a + # guarded no-op that dirties the error slot and races a recycled + # address in other threads). + self.assertEqual(self._free_count(freed, released_handle), 0, + "consumed handle was freed instead of marked consumed") - # close() must not free it again. + # close() must not free it either. builder.close() - self.assertEqual(self._free_count(freed, released_handle), 1, - "close() double-freed an already-released handle") + self.assertEqual(self._free_count(freed, released_handle), 0, + "close() freed a handle already marked consumed") - def test_reader_with_fragment_null_return_frees_self(self): + def test_reader_with_fragment_null_return_marks_consumed(self): init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") with open(init_path, "rb") as init: reader = Reader("video/mp4", init) released_handle = reader._handle - # Mimic an error. + # Mimic a non-tag error: native took ownership then failed and dropped + # the value itself, so the handle is marked consumed, not freed. c2pa_module._lib.c2pa_error_set_last(b"Other: mocked test error") real_call = c2pa_module._lib.c2pa_reader_with_fragment c2pa_module._lib.c2pa_reader_with_fragment = ( lambda r, f, s, frag: None) - # Instrument before the failure so the eager free on the error path is - # counted, not just whatever close() does afterwards. + # Instrument before failure so any free would be counted. freed = self._instrument_frees() try: with open(init_path, "rb") as init, \ @@ -7704,20 +7766,19 @@ def test_reader_with_fragment_null_return_frees_self(self): self.assertIsNone(reader._handle) self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) - # The error path frees the old handle. - self.assertEqual(self._free_count(freed, released_handle), 1, - "error path did not free the old handle exactly once") + # A non-tag error marks consumed: no free. + self.assertEqual(self._free_count(freed, released_handle), 0, + "consumed handle was freed instead of marked consumed") - # close() must not free it again. + # close() must not free it either. reader.close() - self.assertEqual(self._free_count(freed, released_handle), 1, - "close() freed a handle the error path already freed") + self.assertEqual(self._free_count(freed, released_handle), 0, + "close() freed a handle already marked consumed") def test_reader_with_fragment_ffi_raise_frees_self(self): # If the ctypes call itself raises, the failure runs # through the except BaseException branch, which frees the handle. - # with_fragment must free exactly once and leave nothing for - # close() to double-free. + # with_fragment must free exactly once and leave nothing for close(). init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") with open(init_path, "rb") as init: @@ -7752,9 +7813,9 @@ def _raise(*_args): self.assertEqual(self._free_count(freed, released_handle), 1, "close() freed a handle the error path already freed") - # Consume-and-return ownership: the native call takes the handle partway - # through its body, so a null return does not say on its own whether the - # handle was consumed. These pin the classification down. + # Consume-and-return ownership: the native call can take the handle partway + # through the call, so a null return does not always say on its own + # whether the handle was consumed or not, warranting further checks/bookkeeping. @staticmethod def _is_pre_consume_rejection(error_message): @@ -7789,8 +7850,8 @@ def _untracked_reader_handle(): buf) def test_with_fragment_pre_consume_rejection_keeps_handle(self): - # Rejected before Box::from_raw, so nothing was consumed and the - # handle is still ours. Dropping it here would leak it. + # Rejected before native lib took ownership, + # so nothing was consumed and the handle is still ours. init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") with open(init_path, "rb") as init: @@ -7830,15 +7891,14 @@ def test_with_fragment_pre_consume_rejection_does_not_leak(self): reader.with_fragment("video/mp4", init, frag) finally: reader._handle = real_handle - # Still owns a working handle every time round, so nothing - # leaked: a dropped handle would leave the reader closed. + # Still owns a working handle every time round. self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) self.assertTrue(reader.json()) reader.close() def test_with_archive_post_consume_failure_consumes_handle(self): - # Ownership taken, then the operation failed: the handle is gone, - # so close() must not free it again. + # Ownership taken, then the operation failed: + # The handle is gone, so close() must not free it again. builder = Builder(json.dumps( {"claim_generator_info": [{"name": "test", "version": "0.1"}], "assertions": []})) @@ -7856,8 +7916,7 @@ def test_with_archive_post_consume_failure_consumes_handle(self): "close() freed a handle the FFI already consumed") def test_with_fragment_marshalling_error_keeps_handle(self): - # Never reaches native code, so nothing was consumed. The old - # blanket except marked it consumed here and leaked the handle. + # Never reaches native code, so nothing was consumed. init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") with open(init_path, "rb") as init: reader = Reader("video/mp4", init) @@ -8505,8 +8564,8 @@ def test_check_ffi_operation_result_passes_success_through(self): c2pa_module._check_ffi_operation_result(42, "unused"), 42) def test_stream_creation_failure_reports_a_real_message(self): - # Regression: used to raise bare Exception("...: None"), because - # _parse_operation_result_for_error never returns a message. + # Regression: used to raise bare Exception("...: None") instead of the + # native error message. real = c2pa_module._lib.c2pa_create_stream c2pa_module._lib.c2pa_create_stream = lambda *a: None try: From 1e72d836bb34a7ed4f8dc1b9bfdb3cfb274fd025 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:44:58 -0700 Subject: [PATCH 24/67] ci: Merge commit for realz --- docs/native-resources-management.md | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index e21290bd..f5c453dc 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -3,11 +3,7 @@ `ManagedResource` is the internal base class used by the C2PA Python SDK to wrap native (Rust/FFI) pointers. When adding new wrappers around native resources `ManagedResource` should be subclassed and follow the documented lifecycle rules. > [!NOTE] -<<<<<<< HEAD -> `ManagedResource` and the lifecycle machinery described here are internal to the SDK, not part of its public API. In most cases, code that reads and writes C2PA data should use the public wrappers (`Reader`, `Builder`, `Signer`, `Context`, `Settings`). -======= > `ManagedResource` and the lifecycle machinery described here are internal to the SDK. In most cases, code that reads and writes C2PA data should use the public wrappers (`Reader`, `Builder`, `Signer`, `Context`, `Settings`). ->>>>>>> refs/remotes/origin/mathern/open-up-api ## Why `ManagedResource`? @@ -116,11 +112,7 @@ def _free_native_ptr(ptr): _lib.c2pa_free(ptr) ``` -<<<<<<< HEAD -All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). No explicit `ctypes.cast` is needed: `c2pa_free`'s declared argtype is `c_void_p`, so ctypes converts any pointer instance on the way in. Casting explicitly with `ctypes.cast(ptr, c_void_p)` performs the same conversion but leaves a reference cycle behind on every call, so avoid it here. -======= All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). No explicit `ctypes.cast` is needed: `c2pa_free`'s declared argtype is `c_void_p`, so ctypes converts any pointer instance on the way in. Casting explicitly with `ctypes.cast(ptr, c_void_p)` performs the same conversion but leaves a reference cycle behind on every call, which creates additional load on the (Python) garbage collector. ->>>>>>> refs/remotes/origin/mathern/open-up-api `ManagedResource` guarantees that `c2pa_free` is called exactly once per pointer: not zero times (leak), not twice (double-free). @@ -149,12 +141,8 @@ Each transition has one method that performs it, and subclasses must go through | --- | --- | --- | | `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | | `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | -<<<<<<< HEAD -| `_mark_consumed()` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side. Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | -======= | `_mark_consumed()` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | | `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly and closes the object. Same post-state as `_mark_consumed()`. | ->>>>>>> refs/remotes/origin/mathern/open-up-api Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. @@ -349,11 +337,7 @@ stateDiagram-v2 end note ``` -<<<<<<< HEAD -The object stays `ACTIVE` throughout because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. The old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. -======= On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage below. ->>>>>>> refs/remotes/origin/mathern/open-up-api ### `_consume_and_swap()` @@ -370,16 +354,6 @@ The call is passed as a lambda because the helper supplies the handle and, on su The helper exists because a null return can be ambiguous. The native function validates the borrowed pointer first, then takes ownership, then does the work, so a null result can mean either "rejected your pointer, never took it" or "took your pointer, then failed". The native error message is what tells them apart: -<<<<<<< HEAD -| Native error | Who owns the handle | -| --- | --- | -| `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved, so the handle is kept and the resource stays `ACTIVE` | -| Any other error | Assumed taken, so `_mark_consumed()` runs and the resource goes `CLOSED`. The raised error is typed from the native message. | -| No error at all | Same ownership conclusion, but nothing to type the error from, so the caller's message is raised with `"Unknown error"` filled in. | - -This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. - -======= | Native error | Who owns the handle | What the helper does | | --- | --- | --- | | `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | @@ -392,7 +366,6 @@ This triage relies on the native error still being readable after the call retur The binding targets one native contract, the released C FFI (`c2pa-v0.90.0`). Its consuming calls reject a borrowed pointer up front with a `_PRE_CONSUME_ERROR_TAGS` tag (handle retained), or take ownership and, on any later failure, drop the value themselves. So a non-tag error means the value is already gone: `_mark_consumed()` is exact, and a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address in another thread. `_release_handle()` stays only where ownership is genuinely unknown — an async exception mid-call, or the no-error fallthrough no released path produces — where the guarded free is the right default (a real free if the handle is ours, a `-1` no-op if not). The native-side details are not visible from this repo (the library is a prebuilt binary), so treat the contract above as the assumed native behaviour this code is written against. ->>>>>>> refs/remotes/origin/mathern/open-up-api ### Adopting the handle before giving it away `Reader._init_from_context` and `Builder._init_from_context` both create a native object, immediately `_activate()` it, and only then make the consuming call. Reduced to its shape: From d1f5abc3ac68aab1614d3b92588727520f72f7c6 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:50:28 -0700 Subject: [PATCH 25/67] ci: Clarify comments --- src/c2pa/c2pa.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 04e65af9..2e6d5c29 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -307,8 +307,7 @@ def _release(self): """ def _safe_release(self): - """Run _release(), logging and swallowing ordinary errors so teardown - continues. Interrupts (KeyboardInterrupt, SystemExit) propagate. + """Run _release(), logging on error. """ try: self._release() @@ -321,10 +320,7 @@ def _safe_release(self): def _teardown(self, free_handle: bool): """Close the object: run _release, optionally free the handle, null it. - - The one place that owns the foreign-process rule, the CLOSED ordering - and the guarded free. free_handle=False (consumed) frees nothing; the - new owner does. + free_handle=False (consumed) frees nothing, the new owner needs to free. """ if is_foreign_process(self): self._handle = None @@ -335,18 +331,15 @@ def _teardown(self, free_handle: bool): try: self._safe_release() finally: - # Free in finally so an interrupt escaping _release() (a BaseException - # _safe_release does not swallow) still frees an owned handle. State - # is already CLOSED, so no later teardown path would retry it. + # In finally: an interrupt escaping _release() still frees the + # handle (state is CLOSED, so no later path would retry). handle, self._handle = self._handle, None if free_handle and handle: try: ManagedResource._free_native_ptr(handle) except Exception: - logger.error( - "Failed to free native %s resources", - type(self).__name__, - exc_info=True) + logger.error("Failed to free native %s resources", + type(self).__name__, exc_info=True) def _release_handle(self): """Free this handle, then close the object. Used only where ownership is @@ -413,15 +406,11 @@ def _swap_handle(self, new_handle): def _invoke_consume(self, ffi_call, error_message): """Run an FFI call that consumes this handle, returning its raw result. - A marshalling ArgumentError is re-raised untouched: the call never - reached the native side, so the handle is still ours and unchanged. An - ordinary Exception from the call frees the handle before raising; an - interrupt (KeyboardInterrupt/SystemExit) propagates untouched and the - still-ACTIVE handle is freed later at close()/GC (a guarded free). - - The caller inspects the returned result to tell success from failure - (the convention differs per call) and routes a failure to - _raise_consume_failure. + A marshalling ArgumentError is re-raised untouched (call never reached + native, handle unchanged). An Exception frees the handle before + raising. The caller inspects the returned result to tell success + from failure (the convention differs per call) and routes a failure + to _raise_consume_failure. Args: ffi_call: Callable taking the current handle, returning the native From 2ba7209c62add060bbaae4499385b7ca58e0678a Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:52:56 -0700 Subject: [PATCH 26/67] ci: Re-refactor --- src/c2pa/c2pa.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 2e6d5c29..434a1982 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -328,18 +328,15 @@ def _teardown(self, free_handle: bool): return self._lifecycle_state = LifecycleState.CLOSED - try: - self._safe_release() - finally: - # In finally: an interrupt escaping _release() still frees the - # handle (state is CLOSED, so no later path would retry). - handle, self._handle = self._handle, None - if free_handle and handle: - try: - ManagedResource._free_native_ptr(handle) - except Exception: - logger.error("Failed to free native %s resources", - type(self).__name__, exc_info=True) + self._safe_release() + + handle, self._handle = self._handle, None + if free_handle and handle: + try: + ManagedResource._free_native_ptr(handle) + except Exception: + logger.error("Failed to free native %s resources", + type(self).__name__, exc_info=True) def _release_handle(self): """Free this handle, then close the object. Used only where ownership is From 61d503c6dea273b6f17ccea3cdef26f48e4171d0 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:55:50 -0700 Subject: [PATCH 27/67] fix: Refactor + docs update --- docs/native-resources-management.md | 72 +++++++++++++++------------ src/c2pa/c2pa.py | 76 +++++++++++------------------ 2 files changed, 71 insertions(+), 77 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index f5c453dc..f06590ca 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -90,8 +90,8 @@ Notes: | **Cleanup is idempotent** | Calling `close()` (or exiting a `with` block) multiple times is safe; after the first successful cleanup, further calls do nothing. | | **Cleanup never raises** | The cleanup path is wrapped so that exceptions are caught and logged, never re-raised. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. | | **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED to ACTIVE to CLOSED. A closed resource cannot be reactivated. | -| **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_mark_consumed()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | -| **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_mark_consumed()`), the object stops managing it and does not call `c2pa_free` on it. | +| **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_teardown()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | +| **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_teardown(free_handle=False)`), the object stops managing it and does not call `c2pa_free` on it. | | **Public methods validate lifecycle state** | Every public API calls `_ensure_valid_state()` before use; closed or invalid state yields `C2paError` instead of undefined behavior or crashes. | ## Preventing garbage collection of live references @@ -109,11 +109,13 @@ The native Rust library exposes a single C FFI function, `c2pa_free`, that deall ```python @staticmethod def _free_native_ptr(ptr): - _lib.c2pa_free(ptr) + return _lib.c2pa_free(ptr) ``` All native pointers are freed through this single path, regardless of which constructor created them (`c2pa_reader_from_stream`, `c2pa_builder_from_json`, `c2pa_signer_from_info`, etc.). No explicit `ctypes.cast` is needed: `c2pa_free`'s declared argtype is `c_void_p`, so ctypes converts any pointer instance on the way in. Casting explicitly with `ctypes.cast(ptr, c_void_p)` performs the same conversion but leaves a reference cycle behind on every call, which creates additional load on the (Python) garbage collector. +It returns `c2pa_free`'s status code: `0` when the pointer was really freed, `-1` when the native registry rejected an already-consumed or untracked address. That `-1` is expected on the guarded-free paths and is handled gracefully by the native lib too. + `ManagedResource` guarantees that `c2pa_free` is called exactly once per pointer: not zero times (leak), not twice (double-free). ## Lifecycle states @@ -125,15 +127,16 @@ stateDiagram-v2 direction LR [*] --> UNINITIALIZED : __init__() UNINITIALIZED --> ACTIVE : _activate(handle) + UNINITIALIZED --> CLOSED : close() before activation ACTIVE --> ACTIVE : _swap_handle(new_handle) - ACTIVE --> CLOSED : close() / __exit__ / __del__ / _mark_consumed() + ACTIVE --> CLOSED : close() / __exit__ / __del__ / _teardown() ``` - `UNINITIALIZED`: The Python object exists but the native pointer has not been set yet. This is a transient state during construction. - `ACTIVE`: The native pointer is valid. The object can be used. - `CLOSED`: The native pointer has been freed (or ownership was transferred). Any further use raises `C2paError`. -The transition from ACTIVE to CLOSED is one-way. Once closed, an object cannot be reactivated. +`CLOSED` is a one-way state: once closed, an object cannot be reactivated. It is normally reached from `ACTIVE`, but a construction that fails before `_activate()` closes straight from `UNINITIALIZED` when `close()` or `__del__` runs (nothing to free, just marked closed). Each transition has one method that performs it, and subclasses must go through them rather than assigning `_handle` or `_lifecycle_state` directly: @@ -141,11 +144,18 @@ Each transition has one method that performs it, and subclasses must go through | --- | --- | --- | | `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | | `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | -| `_mark_consumed()` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | -| `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly and closes the object. Same post-state as `_mark_consumed()`. | +| `_teardown(free_handle=False)` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | +| `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly (via `_teardown(free_handle=True)`) and closes the object. Same post-state as the consumed teardown. | Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. +`_teardown(free_handle)` is the one method that performs the ACTIVE to CLOSED transition, and the boolean decides the only thing that varies between the two exit paths: whether the native pointer is freed. Both paths run `_release()`, set `CLOSED`, and null the handle. + +| `free_handle` | When | What it does with the pointer | +| --- | --- | --- | +| `True` | We still own the pointer: normal `close()`, `__del__`, or a failure where ownership is unknown (`_release_handle()`). | Calls `c2pa_free` (guarded). | +| `False` | The native side already took ownership: a consuming FFI call swallowed the pointer, or it passed to another object. | Frees nothing; the new owner does. A `c2pa_free` here would double-free (or hit the guarded `-1` no-op that dirties the error slot and risks racing a recycled address). | + Every public method calls `_ensure_valid_state()` before doing any work, which raises `C2paError` unless the resource is ACTIVE with a non-null handle. ## Ways to clean up @@ -261,13 +271,13 @@ stateDiagram-v2 While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` closes the Builder when it returns, on both the success and the failure path. Closing without signing frees the pointer the same way. -The native sign call borrows the builder's pointer rather than taking ownership of it, so `Builder` never calls `_mark_consumed()` and the pointer is freed normally through `c2pa_free`. The close enforces single use; it is not a memory-management requirement. +The native sign call borrows the builder's pointer rather than taking ownership of it, so `Builder` never marks it consumed and the pointer is freed normally through `c2pa_free`. The close enforces single use; it is not a memory-management requirement. ## Ownership transfer Some operations transfer a native pointer from one object to another. When this happens, the original object must stop managing the pointer (e.g. so it is not freed twice). -`_mark_consumed()` handles this. It sets `_handle = None` and `_lifecycle_state = CLOSED` in one step. +`_teardown(free_handle=False)` handles this. It runs `_release()`, then sets `_handle = None` and `_lifecycle_state = CLOSED` without freeing the pointer. In the SDK this happens in one place: passing a `Signer` to a `Context`. The Context takes ownership of the Signer's native pointer, and the Signer must not be used again directly after that. @@ -291,7 +301,7 @@ sequenceDiagram X-->>C: re-raise, Signer keeps its handle else call returned N-->>X: result - X->>S: _mark_consumed() + X->>S: _teardown(free_handle=False) Note right of S: Unconditional, before checking result:
set_signer takes ownership either way X->>X: raise if result != 0 end @@ -303,8 +313,8 @@ sequenceDiagram Details in that sequence that are easy to get wrong: -- The callback is copied to the Context *before* the transfer. `_mark_consumed()` runs `_release()`, so consuming the Signer drops its reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. -- `_mark_consumed()` runs before the result is checked, not after. A failed `set_signer` has still taken the pointer, so waiting for a successful result would leak it. +- The callback is copied to the Context *before* the transfer. The consumed teardown runs `_release()`, so consuming the Signer drops its reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. +- The consumed teardown runs before the result is checked, not after. A failed `set_signer` has still taken the pointer, so waiting for a successful result would leak it. - `ctypes.ArgumentError` is the exception. It means ctypes could not marshal the arguments, so the native function never ran and never saw the pointer. The Signer still owns its handle and is left untouched. Both `c2pa_context_builder_set_signer` and `c2pa_context_builder_build` consume what they are given, so the `builder_ptr` is freed by the error handler only when the build was never reached. @@ -319,7 +329,7 @@ Ownership transfers only if the call returns. If `_wrap_native_handle()` raises, ## Consume-and-swap -`_mark_consumed()` closes an object permanently. A different pattern is needed when the native library must replace an object's internal state without discarding the Python-side object. This happens with fragmented media: `Reader.with_fragment()` feeds a new BMFF fragment (used in DASH/HLS streaming) into an existing Reader, and the native library must rebuild its internal representation to account for the new data. The native API does this by consuming the old pointer and returning a new one. Creating a fresh `Reader` from scratch would not work because the native library needs the accumulated state from prior fragments. +`_teardown(free_handle=False)` closes an object permanently. A different pattern is needed when the native library must replace an object's internal state without discarding the Python-side object. This happens with fragmented media: `Reader.with_fragment()` feeds a new BMFF fragment (used in DASH/HLS streaming) into an existing Reader, and the native library must rebuild its internal representation to account for the new data. The native API does this by consuming the old pointer and returning a new one. Creating a fresh `Reader` from scratch would not work because the native library needs the accumulated state from prior fragments. `Builder.with_archive()` follows the same pattern: it loads an archive into an existing Builder, replacing the manifest definition while preserving the Builder's context and settings. @@ -337,7 +347,7 @@ stateDiagram-v2 end note ``` -On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from `_mark_consumed()`, where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage below. +On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from a consumed teardown (`_teardown(free_handle=False)`), where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage below. ### `_consume_and_swap()` @@ -357,21 +367,23 @@ The helper exists because a null return can be ambiguous. The native function va | Native error | Who owns the handle | What the helper does | | --- | --- | --- | | `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | -| Any other error | Taken, then the operation failed | `_mark_consumed()`: the native side already dropped the value, so nothing is freed here; resource goes `CLOSED`, error typed from the native message. | +| Any other error | Taken, then the operation failed | `_teardown(free_handle=False)`: the native side already dropped the value, so nothing is freed here; resource goes `CLOSED`, error typed from the native message. | | No error at all | Unknown (no released path reaches here) | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. #### Why a non-tag error does not free -The binding targets one native contract, the released C FFI (`c2pa-v0.90.0`). Its consuming calls reject a borrowed pointer up front with a `_PRE_CONSUME_ERROR_TAGS` tag (handle retained), or take ownership and, on any later failure, drop the value themselves. So a non-tag error means the value is already gone: `_mark_consumed()` is exact, and a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address in another thread. `_release_handle()` stays only where ownership is genuinely unknown — an async exception mid-call, or the no-error fallthrough no released path produces — where the guarded free is the right default (a real free if the handle is ours, a `-1` no-op if not). The native-side details are not visible from this repo (the library is a prebuilt binary), so treat the contract above as the assumed native behaviour this code is written against. +The binding targets one native contract, the released C FFI (`c2pa-v0.90.0`). Its consuming calls reject a borrowed pointer up front with a `_PRE_CONSUME_ERROR_TAGS` tag (handle retained), or take ownership and, on any later failure, drop the value themselves. So a non-tag error means the value is already gone: `_teardown(free_handle=False)` is exact, and a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address in another thread. `_release_handle()` stays only where ownership is genuinely unknown — an async exception mid-call, or the no-error fallthrough no released path produces — where the guarded free is the right default (a real free if the handle is ours, a `-1` no-op if not). The native-side details are not visible from this repo (the library is a prebuilt binary), so treat the contract above as the assumed native behaviour this code is written against. ### Adopting the handle before giving it away -`Reader._init_from_context` and `Builder._init_from_context` both create a native object, immediately `_activate()` it, and only then make the consuming call. Reduced to its shape: +`Reader._init_from_context` and `Builder._init_from_context` both create a native object, immediately activate it, and only then make the consuming call. `_create_and_activate()` handles the create-then-activate half: it calls the FFI constructor, validates the result with `_check_ffi_operation_result`, and `_activate()`s it, freeing the pointer if either step fails so a rejected creation leaks nothing. Reduced to its shape: ```python -self._activate(reader_ptr) +self._create_and_activate( + lambda: _lib.c2pa_reader_from_context(context.execution_context), + Reader._ERROR_MESSAGES['reader_error']) self._consume_and_swap( lambda handle: _lib.c2pa_reader_with_stream( @@ -433,7 +445,7 @@ sequenceDiagram P->>P: frees the native pointer normally ``` -Both `_cleanup_resources()` and `_mark_consumed()` take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. +Both `_cleanup_resources()` and the consumed teardown take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. The memory the child skips is not lost for good. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. @@ -469,17 +481,17 @@ class NativeResource(ManagedResource): super().__init__() self._init_attrs() - # 2. Create the native pointer. Pass the error message as a - # template: _check_ffi_operation_result fills in the native - # error, or "Unknown error" when there is none. - handle = _lib.c2pa_my_resource_new(arg) - _check_ffi_operation_result(handle, "Failed to create MyResource: {}") - - # 3. Take ownership only after the FFI call succeeded. - # _activate() rejects a null handle and refuses to run twice, - # so the object is never ACTIVE without a live pointer. - # Never assign self._handle or self._lifecycle_state directly. - self._activate(handle) + # 2. Create the native pointer, validate it, and take ownership. + # _create_and_activate() runs the FFI call, checks the result + # with _check_ffi_operation_result (which fills in the native + # error, or "Unknown error" when there is none), then _activate()s + # it. If any step fails the pointer is freed, so a rejected + # creation leaks nothing. The object is never ACTIVE without a + # live pointer. Never assign self._handle or self._lifecycle_state + # directly. + self._create_and_activate( + lambda: _lib.c2pa_my_resource_new(arg), + "Failed to create MyResource: {}") def _release(self): # 4. Clean up class-specific resources. diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 434a1982..0217a9f7 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -371,6 +371,22 @@ def _activate(self, handle): self._handle = handle self._lifecycle_state = LifecycleState.ACTIVE + def _create_and_activate(self, ffi_call, error_message, *, + check=lambda r: not r): + """Obtain a fresh native pointer, validate it, and take ownership. + On any failure before ownership transfers, the pointer is freed + and the error re-raised. + """ + ptr = ffi_call() + try: + _check_ffi_operation_result(ptr, error_message, check=check) + self._activate(ptr) + except Exception: + if ptr: + ManagedResource._free_native_ptr(ptr) + raise + return ptr + def _swap_handle(self, new_handle): """Replace the handle after an FFI call consumed the old one and returned a replacement. @@ -1504,16 +1520,8 @@ def __init__(self): """Create new Settings with default values.""" super().__init__() - settings_ptr = _lib.c2pa_settings_new() - try: - _check_ffi_operation_result( - settings_ptr, "Failed to create Settings") - except Exception: - if settings_ptr: - ManagedResource._free_native_ptr(settings_ptr) - raise - - self._activate(settings_ptr) + self._create_and_activate( + _lib.c2pa_settings_new, "Failed to create Settings") @classmethod def from_json(cls, json_str: str) -> 'Settings': @@ -2489,25 +2497,12 @@ def _init_from_context(self, context, format_or_path, self._own_stream = Stream(stream) try: - # Create reader from context - reader_ptr = _lib.c2pa_reader_from_context( - context.execution_context, - ) - try: - _check_ffi_operation_result(reader_ptr, - Reader._ERROR_MESSAGES[ - 'reader_error' - ] - ) - except Exception: - if reader_ptr: - ManagedResource._free_native_ptr(reader_ptr) - raise - - # Adopt the handle before the consuming call: _consume_and_swap - # needs an active resource, and from here on normal cleanup owns - # the pointer whichever way the call goes. - self._activate(reader_ptr) + # Adopt before the consuming call: _consume_and_swap needs an + # active resource, and cleanup then owns the pointer either way. + self._create_and_activate( + lambda: _lib.c2pa_reader_from_context( + context.execution_context), + Reader._ERROR_MESSAGES['reader_error']) if manifest_data is not None: manifest_array = ( @@ -3276,24 +3271,11 @@ def _init_from_context(self, context, json_str): if not context.is_valid: raise C2paError("Context is not valid") - builder_ptr = _lib.c2pa_builder_from_context( - context.execution_context, - ) - try: - _check_ffi_operation_result(builder_ptr, - Builder._ERROR_MESSAGES[ - 'builder_error' - ] - ) - except Exception: - if builder_ptr: - ManagedResource._free_native_ptr(builder_ptr) - raise - - # Adopt the handle before the consuming call: - # _consume_and_swap needs an active resource, - # and from here on normal cleanup owns the pointer. - self._activate(builder_ptr) + # Adopt before the consuming call: _consume_and_swap needs an + # active resource, and cleanup then owns the pointer either way. + self._create_and_activate( + lambda: _lib.c2pa_builder_from_context(context.execution_context), + Builder._ERROR_MESSAGES['builder_error']) self._consume_and_swap( lambda handle: _lib.c2pa_builder_with_definition( From e857a521ba1a1d6030d4590ee0a5813ae44dff3b Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:11:32 -0700 Subject: [PATCH 28/67] fix: Docs --- docs/native-resources-management.md | 67 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index f06590ca..be0ff87b 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -88,7 +88,7 @@ Notes: | --- | --- | | **Pointer freed exactly once** | Each native pointer is passed to `c2pa_free` at most once. No leak (zero frees) and no double-free. | | **Cleanup is idempotent** | Calling `close()` (or exiting a `with` block) multiple times is safe; after the first successful cleanup, further calls do nothing. | -| **Cleanup never raises** | The cleanup path is wrapped so that exceptions are caught and logged, never re-raised. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. | +| **Cleanup never raises (ordinary errors)** | The cleanup path catches and logs `Exception`, never re-raising it. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. **Interrupts are the deliberate exception:** `KeyboardInterrupt` / `SystemExit` are `BaseException`, not `Exception`, so they propagate out of cleanup and may skip the remaining free. This is intentional — an interrupt kills the process and the OS reclaims the memory, so swallowing it to guard a few bytes would be worse than letting it through. Do not widen these handlers to `except BaseException`. | | **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED to ACTIVE to CLOSED. A closed resource cannot be reactivated. | | **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_teardown()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | | **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_teardown(free_handle=False)`), the object stops managing it and does not call `c2pa_free` on it. | @@ -188,14 +188,16 @@ If neither of the above is used, `__del__` attempts to free the native pointer w ## Error handling during cleanup -Cleanup must never raise an exception. A failure during cleanup (for example, the native library crashing on free) should not mask the original exception that caused the `with` block to exit. `ManagedResource` enforces this: +Cleanup must not raise an *ordinary* exception. A failure during cleanup (for example, the native library crashing on free) should not mask the original exception that caused the `with` block to exit. `ManagedResource` enforces this: -- `close()` delegates to `_cleanup_resources()`, which wraps the entire cleanup sequence in a try/except that catches and silences all exceptions. -- `_release()` is never called directly during cleanup. It runs inside `_safe_release()`, which logs any failure with a traceback and returns normally, so a subclass whose `_release()` raises cannot stop the native pointer from being freed afterwards. +- `close()` delegates to `_cleanup_resources()`, which wraps the entire cleanup sequence in a try/except that catches and silences `Exception`. +- `_release()` is never called directly during cleanup. It runs inside `_safe_release()`, which logs any `Exception` with a traceback and returns normally, so a subclass whose `_release()` raises an ordinary error cannot stop the native pointer from being freed afterwards. - If freeing the native pointer fails, the error is logged via Python's `logging` module but not re-raised. - The state is set to `CLOSED` as the very first step, before attempting to free anything. If cleanup fails halfway, the object is still marked closed, preventing a second attempt from doing further damage. - Cleanup is idempotent. Calling `close()` on an already-closed object returns immediately. +These handlers catch `Exception`, not `BaseException`. A `SystemExit` raised inside cleanup (for instance during a slow `_release()`) propagates out and may skip the remaining free. The interrupt is tearing the process down anyway and the OS will reclaim the memory. + All three cleanup entry points converge on the same method, and the exception handling sits at three different levels inside it: ```mermaid @@ -279,45 +281,48 @@ Some operations transfer a native pointer from one object to another. When this `_teardown(free_handle=False)` handles this. It runs `_release()`, then sets `_handle = None` and `_lifecycle_state = CLOSED` without freeing the pointer. -In the SDK this happens in one place: passing a `Signer` to a `Context`. The Context takes ownership of the Signer's native pointer, and the Signer must not be used again directly after that. +In the SDK this happens in one place: passing a `Signer` to a `Context`. The Context runs a short-lived native context builder, feeds the signer into it, builds the context, and activates the result. The builder itself is wrapped in `_NativeBuilder` (a small `ManagedResource`), so every failure inside the `with` block frees it through `close()` unless a consuming call already took it. There is no raw pointer held across the calls and no bespoke error handler. -The order of operations matters, because `c2pa_context_builder_set_signer` takes ownership on its error path as well as on success: +The signer transfer goes through `_consume_no_replacement()`, which routes any failure to the shared triage (see [Consume-and-swap](#consume-and-swap)). That triage decides per error whether the signer was actually consumed: `set_signer` does *not* unconditionally take ownership. ```mermaid sequenceDiagram participant C as Caller participant S as Signer participant X as Context + participant B as _NativeBuilder participant N as Native lib C->>X: Context(settings, signer) + X->>B: with _NativeBuilder() (owns the builder; close() frees it on any failure) X->>S: _ensure_valid_state() X->>X: copy signer._callback_cb to _signer_callback_cb - Note right of X: Pin the callback first:
the Signer is about to close - X->>N: c2pa_context_builder_set_signer(builder_ptr, handle) - - alt ctypes.ArgumentError - Note over X,N: Marshalling failed, native side never ran - X-->>C: re-raise, Signer keeps its handle - else call returned - N-->>X: result - X->>S: _teardown(free_handle=False) - Note right of S: Unconditional, before checking result:
set_signer takes ownership either way - X->>X: raise if result != 0 + Note right of X: Pin the callback first:
the Signer is about to be consumed + X->>S: _consume_no_replacement(set_signer) + S->>N: c2pa_context_builder_set_signer(builder_ptr, handle) + + alt status 0 (success) + S->>S: _teardown(free_handle=False) + Note right of S: Consumed: native took the signer + else pre-consume tag (UntrackedPointer / WrongPointerType) + Note right of S: Rejected before ownership moved:
Signer retained, typed error raised + else other error + S->>S: _teardown(free_handle=False) + Note right of S: Native took it then failed and dropped it end - X->>N: c2pa_context_builder_build(builder_ptr) - N-->>X: context_ptr - X->>X: _activate(context_ptr) + X->>B: _consume_into(build) + B->>N: c2pa_context_builder_build(builder_ptr) + N-->>X: context_ptr (builder consumed) + X->>X: _activate(context_ptr) (outside the with) ``` Details in that sequence that are easy to get wrong: -- The callback is copied to the Context *before* the transfer. The consumed teardown runs `_release()`, so consuming the Signer drops its reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. -- The consumed teardown runs before the result is checked, not after. A failed `set_signer` has still taken the pointer, so waiting for a successful result would leak it. -- `ctypes.ArgumentError` is the exception. It means ctypes could not marshal the arguments, so the native function never ran and never saw the pointer. The Signer still owns its handle and is left untouched. - -Both `c2pa_context_builder_set_signer` and `c2pa_context_builder_build` consume what they are given, so the `builder_ptr` is freed by the error handler only when the build was never reached. +- The callback is copied to the Context *before* the transfer. A successful consume runs `_release()`, which drops the Signer's reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. +- `set_signer` does not always take the pointer. A pre-consume rejection (`UntrackedPointer:` / `WrongPointerType:`) leaves the Signer `ACTIVE` and retained, so the triage must read the native error before deciding to close it. Treating every failure as "consumed" would close a signer the native side never took. +- A `ctypes.ArgumentError` from `set_signer` is re-raised untouched by `_invoke_consume`: marshalling failed, the native function never ran, and the Signer still owns its handle. Only calls that reached native go through the consumed/retained triage. +- The builder is never held as a raw local across the signer and build calls. `_NativeBuilder`'s `with` block owns it: a settings error, a retained-signer error, a build rejection, or an async interrupt all free it through `close()`, and a successful build consumes it so `close()` is then a no-op. The old raw-pointer recovery block that used to free `builder_ptr` on the un-reached-build path is gone. ### Adopting a handle the SDK already owns @@ -372,6 +377,16 @@ The helper exists because a null return can be ambiguous. The native function va This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. +Three consume helpers share this triage; they differ only in what the FFI call returns on success: + +| Helper | Success return | Success action | +| --- | --- | --- | +| `_consume_and_swap()` | a replacement pointer | `_swap_handle()`, resource stays `ACTIVE` | +| `_consume_no_replacement()` | a status code (`0` = ok) | `_teardown(free_handle=False)`, resource `CLOSED` | +| `_consume_into()` | a *different* object's pointer | `_teardown(free_handle=False)`, the pointer returned for the caller to own | + +`_consume_no_replacement()` is how a `Signer` is fed to a `Context` (`set_signer` returns a status code); `_consume_into()` is how that same `Context` build returns the new context pointer. Any failure in any of the three lands in the table above, so a pre-consume rejection retains the handle rather than assuming it was taken. + #### Why a non-tag error does not free The binding targets one native contract, the released C FFI (`c2pa-v0.90.0`). Its consuming calls reject a borrowed pointer up front with a `_PRE_CONSUME_ERROR_TAGS` tag (handle retained), or take ownership and, on any later failure, drop the value themselves. So a non-tag error means the value is already gone: `_teardown(free_handle=False)` is exact, and a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address in another thread. `_release_handle()` stays only where ownership is genuinely unknown — an async exception mid-call, or the no-error fallthrough no released path produces — where the guarded free is the right default (a real free if the handle is ours, a `-1` no-op if not). The native-side details are not visible from this repo (the library is a prebuilt binary), so treat the contract above as the assumed native behaviour this code is written against. @@ -416,7 +431,7 @@ The cleanup order matters: `_release()` runs first (closing streams, dropping ca Clearing it in `_release()` is the other half of that. A closed Reader has no further use for the Context, and holding the reference would keep alive an object nothing can reach through the Reader's public API. -Dropping the reference before the native pointer is freed is safe because the native side does not depend on the Python object staying alive. `Reader::from_shared_context` clones the underlying `Arc`, so the native reader holds its own count on the context and does not care whether Python still points at it. +Dropping the reference before the native pointer is freed is safe because the native side does not depend on the Python object staying alive. When a reader is created from a shared context, the native side takes its own reference-counted handle on that context (an `Arc` clone in the Rust library), so the native reader keeps the context alive independently of whether Python still points at it. (That native behaviour is not visible from this repo, which ships a prebuilt binary; it is the contract this code is written against.) ## Fork safety From 1ee9a49b7d44f9337eb7a88ec2d7c13c60d7358b Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:18:22 -0700 Subject: [PATCH 29/67] fix: Docs 2 --- docs/native-resources-management.md | 36 +++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index be0ff87b..6e029eb7 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -14,7 +14,7 @@ - Ownership transfers (e.g. signer to context) are handled so the same pointer is not freed twice (and the objects/classes know which one owns what). - Cleanup never raises (trade-off to avoid raising errors on clean-up only, but errors are logged). -Developers wrapping new native resources must inherit from `ManagedResource` and follow the documented lifecycle rules. +A new wrapper around a native resource inherits from `ManagedResource` and follows the documented lifecycle rules. ## Why is native resources management needed? @@ -88,7 +88,7 @@ Notes: | --- | --- | | **Pointer freed exactly once** | Each native pointer is passed to `c2pa_free` at most once. No leak (zero frees) and no double-free. | | **Cleanup is idempotent** | Calling `close()` (or exiting a `with` block) multiple times is safe; after the first successful cleanup, further calls do nothing. | -| **Cleanup never raises (ordinary errors)** | The cleanup path catches and logs `Exception`, never re-raising it. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. **Interrupts are the deliberate exception:** `KeyboardInterrupt` / `SystemExit` are `BaseException`, not `Exception`, so they propagate out of cleanup and may skip the remaining free. This is intentional — an interrupt kills the process and the OS reclaims the memory, so swallowing it to guard a few bytes would be worse than letting it through. Do not widen these handlers to `except BaseException`. | +| **Cleanup never raises (ordinary errors)** | The cleanup path catches and logs `Exception`, never re-raising it. `_release()` runs inside `_safe_release()`, which logs and swallows; the `c2pa_free` call has its own handler; and `_cleanup_resources()` wraps both. The original exception from the `with` block (if any) is never masked. **Asynchronous interrupts are the deliberate exception.** The cleanup handlers catch `Exception`, which excludes the `BaseException` signals the interpreter raises to unwind a process (a cancellation request or an exit in progress). Those propagate through cleanup untouched, and the remaining free may not run. Such a signal means the process is being torn down and its address space, native allocations included, is about to be reclaimed as a whole. Catching it would suppress a shutdown the caller asked for in order to complete a free that is about to become irrelevant, so the handlers stay scoped to `Exception`. | | **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED to ACTIVE to CLOSED. A closed resource cannot be reactivated. | | **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_teardown()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | | **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_teardown(free_handle=False)`), the object stops managing it and does not call `c2pa_free` on it. | @@ -196,7 +196,7 @@ Cleanup must not raise an *ordinary* exception. A failure during cleanup (for ex - The state is set to `CLOSED` as the very first step, before attempting to free anything. If cleanup fails halfway, the object is still marked closed, preventing a second attempt from doing further damage. - Cleanup is idempotent. Calling `close()` on an already-closed object returns immediately. -These handlers catch `Exception`, not `BaseException`. A `SystemExit` raised inside cleanup (for instance during a slow `_release()`) propagates out and may skip the remaining free. The interrupt is tearing the process down anyway and the OS will reclaim the memory. +These handlers catch `Exception`, not `BaseException`. The signals the interpreter raises to unwind a process (a cancellation request, or an exit already in progress) are `BaseException`, so they pass through cleanup untouched and the remaining free may not run. That is intentional: the signal means the whole process is going away, and its address space, native allocations included, is reclaimed on exit. Holding the interpreter in cleanup to finish a free that is about to become irrelevant would only delay the shutdown the caller asked for. All three cleanup entry points converge on the same method, and the exception handling sits at three different levels inside it: @@ -367,7 +367,7 @@ self._consume_and_swap( The call is passed as a lambda because the helper supplies the handle and, on success, replaces it via `_swap_handle()`. -The helper exists because a null return can be ambiguous. The native function validates the borrowed pointer first, then takes ownership, then does the work, so a null result can mean either "rejected your pointer, never took it" or "took your pointer, then failed". The native error message is what tells them apart: +The helper exists because a null return can be ambiguous. The native function validates the borrowed pointer first, then takes ownership, then does the work, so a null result can mean either "rejected the pointer, never took it" or "took the pointer, then failed". The native error message is what tells them apart: | Native error | Who owns the handle | What the helper does | | --- | --- | --- | @@ -387,9 +387,15 @@ Three consume helpers share this triage; they differ only in what the FFI call r `_consume_no_replacement()` is how a `Signer` is fed to a `Context` (`set_signer` returns a status code); `_consume_into()` is how that same `Context` build returns the new context pointer. Any failure in any of the three lands in the table above, so a pre-consume rejection retains the handle rather than assuming it was taken. -#### Why a non-tag error does not free +#### Why an ownership-taken failure does not free -The binding targets one native contract, the released C FFI (`c2pa-v0.90.0`). Its consuming calls reject a borrowed pointer up front with a `_PRE_CONSUME_ERROR_TAGS` tag (handle retained), or take ownership and, on any later failure, drop the value themselves. So a non-tag error means the value is already gone: `_teardown(free_handle=False)` is exact, and a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address in another thread. `_release_handle()` stays only where ownership is genuinely unknown — an async exception mid-call, or the no-error fallthrough no released path produces — where the guarded free is the right default (a real free if the handle is ours, a `-1` no-op if not). The native-side details are not visible from this repo (the library is a prebuilt binary), so treat the contract above as the assumed native behaviour this code is written against. +A consuming FFI call fails in one of two ways. It either rejects the borrowed pointer before taking it, or it takes ownership first and then, on any later failure, drops the value itself. There is no failure path that takes the pointer and leaves it for the caller to free. + +The native error message says which of the two happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS` (`UntrackedPointer:` or `WrongPointerType:`), so the handle was never taken and is retained. Any other error message means the native side took ownership and already dropped the value. + +So a taken-then-failed error means the value is already gone, and `_teardown(free_handle=False)` is exact: a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address. `_release_handle()` (a guarded free) is used only where ownership is unknown, such as an exception mid-call or the no-error fallthrough that no defined failure produces. There the guarded free is the right default: a real free if the handle is ours, a `-1` no-op if not. + +The native side is a prebuilt binary and not visible from this repo, so this two-outcome contract is the behavior the code is written against rather than something it can inspect. ### Adopting the handle before giving it away @@ -431,7 +437,7 @@ The cleanup order matters: `_release()` runs first (closing streams, dropping ca Clearing it in `_release()` is the other half of that. A closed Reader has no further use for the Context, and holding the reference would keep alive an object nothing can reach through the Reader's public API. -Dropping the reference before the native pointer is freed is safe because the native side does not depend on the Python object staying alive. When a reader is created from a shared context, the native side takes its own reference-counted handle on that context (an `Arc` clone in the Rust library), so the native reader keeps the context alive independently of whether Python still points at it. (That native behaviour is not visible from this repo, which ships a prebuilt binary; it is the contract this code is written against.) +Dropping the reference before the native pointer is freed is safe because the native side does not depend on the Python object staying alive. When a reader is created from a shared context, the native side takes its own reference-counted handle on that context (an `Arc` clone in the Rust library), so the native reader keeps the context alive independently of whether Python still points at it. (That native behavior is not visible from this repo, which ships a prebuilt binary; it is the contract this code is written against.) ## Fork safety @@ -535,18 +541,18 @@ class NativeResource(ManagedResource): ### Troubleshooting -- If an attribute is set only in `__init__`, an instance built by `_wrap_native_handle()` will not have it, because that path never runs `__init__`. The failure shows up later as an `AttributeError` from whichever method reads the attribute, often `_release()` during cleanup. Declare attributes in `_init_attrs()` and call it from `__init__`. +- An attribute set only in `__init__` is missing on an instance built by `_wrap_native_handle()`, because that path never runs `__init__`. The failure shows up later as an `AttributeError` from whichever method reads the attribute, often `_release()` during cleanup. Attributes belong in `_init_attrs()`, which `__init__` calls. -- If `_init_attrs()` is called after an FFI call that can raise, and the call fails, `_release()` will access attributes that do not exist yet and crash with `AttributeError`. Call it immediately after `super().__init__()`, before anything that can fail. +- `_init_attrs()` called after an FFI call that can raise leaves `_release()` accessing attributes that do not exist yet when that call fails, crashing with `AttributeError`. It belongs immediately after `super().__init__()`, before anything that can fail. -- Assigning `self._handle` or `self._lifecycle_state` directly bypasses the checks that make the lifecycle safe. `_activate()` refuses a null handle and refuses to run on an already-active object; `_swap_handle()` requires the resource to be active and the replacement non-null. Assigning the fields yourself gives up both, and the resulting bugs (an ACTIVE object with a null handle, or a silently discarded pointer) surface far from their cause. +- Assigning `self._handle` or `self._lifecycle_state` directly bypasses the checks that make the lifecycle safe. `_activate()` refuses a null handle and refuses to run on an already-active object; `_swap_handle()` requires the resource to be active and the replacement non-null. Direct assignment gives up both, and the resulting bugs (an ACTIVE object with a null handle, or a silently discarded pointer) surface far from their cause. -- If `_release()` raises, the exception is silently swallowed by `_cleanup_resources()`. It will not be visible unless logs are checked. Define a lifecycle for managed resources so `_release()` can check whether they need releasing. Wrap the actual release call in try/except as a fallback for unexpected failures. +- A `_release()` that raises has its exception silently swallowed by `_cleanup_resources()`, visible only in the logs. A small lifecycle for managed resources lets `_release()` check whether they need releasing; the actual release call wrapped in try/except is a fallback for unexpected failures. -- `_release()` can be called more than once (via `close()` then `__del__`, or multiple `close()` calls). Make sure it handles being called on an already-cleaned-up object. Setting attributes to `None` after closing them is the standard pattern. +- `_release()` can be called more than once (via `close()` then `__del__`, or multiple `close()` calls), so it must handle being called on an already-cleaned-up object. Setting attributes to `None` after closing them is the standard pattern. -- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the unknown-ownership failure paths can free eagerly without risking a double-free. Still, do not free manually — the lifecycle owns the pointer and bypassing it defeats the state checks. +- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the unknown-ownership failure paths can free eagerly without risking a double-free. A manual free is still wrong — the lifecycle owns the pointer and bypassing it defeats the state checks. -- If a subclass inherits from both `ManagedResource` and an ABC like `ContextProvider`, and both define a property with the same name (e.g. `is_valid`), Python resolves it using the MRO. The parent listed first in the class definition wins. If the ABC is listed first, Python finds the abstract property before the concrete one and raises `TypeError: Can't instantiate abstract class`. Always list the class with the concrete implementation first (e.g. `class Context(ManagedResource, ContextProvider)`, not `class Context(ContextProvider, ManagedResource)`). +- When a subclass inherits from both `ManagedResource` and an ABC like `ContextProvider`, and both define a property with the same name (e.g. `is_valid`), Python resolves it using the MRO. The parent listed first in the class definition wins. With the ABC listed first, Python finds the abstract property before the concrete one and raises `TypeError: Can't instantiate abstract class`. The class with the concrete implementation therefore comes first (e.g. `class Context(ManagedResource, ContextProvider)`, not `class Context(ContextProvider, ManagedResource)`). -- If two parent classes define the same method or property with different concrete implementations, the MRO silently picks the first one. This can cause subtle bugs where the wrong implementation is used. When combining multiple inheritance with shared property names, verify the MRO with `ClassName.__mro__` or `ClassName.mro()` to confirm the expected resolution order. +- When two parent classes define the same method or property with different concrete implementations, the MRO silently picks the first one, which can cause subtle bugs where the wrong implementation is used. With shared property names across multiple inheritance, `ClassName.__mro__` or `ClassName.mro()` confirms the expected resolution order. From e588a6ad472ef9e4f06eff60e16e3bd12a6da194 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:49:32 -0700 Subject: [PATCH 30/67] fix: Docs 3 --- docs/native-resources-management.md | 39 +++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 6e029eb7..da5c289b 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -304,7 +304,7 @@ sequenceDiagram alt status 0 (success) S->>S: _teardown(free_handle=False) Note right of S: Consumed: native took the signer - else pre-consume tag (UntrackedPointer / WrongPointerType) + else pre-consume rejection (UntrackedPointer / WrongPointerType) Note right of S: Rejected before ownership moved:
Signer retained, typed error raised else other error S->>S: _teardown(free_handle=False) @@ -367,15 +367,30 @@ self._consume_and_swap( The call is passed as a lambda because the helper supplies the handle and, on success, replaces it via `_swap_handle()`. -The helper exists because a null return can be ambiguous. The native function validates the borrowed pointer first, then takes ownership, then does the work, so a null result can mean either "rejected the pointer, never took it" or "took the pointer, then failed". The native error message is what tells them apart: +The helper exists because a failed return can be ambiguous. The native functions run in phases: it validates the borrowed pointer, then takes ownership, then does the work. A failure in the first phase and a failure after the second come back to Python as the same value (a null pointer, or a non-zero status), but they leave ownership in opposite places. + +```mermaid +flowchart TD + CALL["FFI call(handle)"] --> V{"validate borrowed handle"} + V -->|invalid| R["reject: handle NOT taken
sets UntrackedPointer / WrongPointerType"] --> F1["returns a failure value
(null, or non-zero status)"] + V -->|valid| TAKE["take ownership of handle"] + TAKE --> WORK{"execute function logic"} + WORK -->|fails| DROP["native drops the value itself
sets some other error"] --> F2["returns a failure value
(null, or non-zero status)"] + WORK -->|succeeds| OK["returns replacement / 0 / new pointer"] + + F1 -.failure value returned to Python.- AMB(["needs to consult error to know failure mode from Python"]) + F2 -.failure value returned to Python.- AMB +``` + +The two failure paths are indistinguishable from the return value alone. Only the native error message set alongside them tells the phases apart: | Native error | Who owns the handle | What the helper does | | --- | --- | --- | | `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | -| Any other error | Taken, then the operation failed | `_teardown(free_handle=False)`: the native side already dropped the value, so nothing is freed here; resource goes `CLOSED`, error typed from the native message. | -| No error at all | Unknown (no released path reaches here) | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | +| Any other error | Taken, then the operation failed | `_teardown(free_handle=False)`: the native side already dropped the value, so nothing is freed here. Resource goes `CLOSED`, error typed from the native message. | +| No error at all | Unknown | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | -This triage relies on the native error still being readable after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it, and the SDK does not clear it before these calls. +This error and ownership triage flow relies on the native error still being readable (and correctly being the last error encountered) after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it. Three consume helpers share this triage; they differ only in what the FFI call returns on success: @@ -385,17 +400,21 @@ Three consume helpers share this triage; they differ only in what the FFI call r | `_consume_no_replacement()` | a status code (`0` = ok) | `_teardown(free_handle=False)`, resource `CLOSED` | | `_consume_into()` | a *different* object's pointer | `_teardown(free_handle=False)`, the pointer returned for the caller to own | -`_consume_no_replacement()` is how a `Signer` is fed to a `Context` (`set_signer` returns a status code); `_consume_into()` is how that same `Context` build returns the new context pointer. Any failure in any of the three lands in the table above, so a pre-consume rejection retains the handle rather than assuming it was taken. +`_consume_no_replacement()` is how a `Signer` is fed to a `Context` (`set_signer` returns a status code); `_consume_into()` is how that same `Context` build returns the new context pointer. A failure in any of the three is handled by the table above, so a pre-consume rejection retains the handle rather than assuming it was taken. #### Why an ownership-taken failure does not free -A consuming FFI call fails in one of two ways. It either rejects the borrowed pointer before taking it, or it takes ownership first and then, on any later failure, drops the value itself. There is no failure path that takes the pointer and leaves it for the caller to free. +A consuming FFI call can fail. It may reject the borrowed pointer before taking it, or it may take ownership first and then, on a later failure, drop the value itself. + +The native error message indicates which of the errors happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS` (`UntrackedPointer:` or `WrongPointerType:`), which means the handle was never taken and is retained. Any other error message means the native side may have taken ownership and already dropped the value. On top of those, a consuming call whose wrapper runs Python (such as a signing callback) can raise a Python exception before native reports anything at all, and that outcome is handled separately. + +The two settled branches each take the exact action their ownership implies. A pre-consume rejection (an error prefixed `UntrackedPointer:` or `WrongPointerType:`) means the handle is still the caller's, so it is retained and freed later by normal cleanup. Any other native error means the value is already gone, so `_teardown(free_handle=False)` runs the Python-side cleanup without freeing anything. -The native error message says which of the two happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS` (`UntrackedPointer:` or `WrongPointerType:`), so the handle was never taken and is retained. Any other error message means the native side took ownership and already dropped the value. +Always calling the guarded free instead, even where the value is known to be gone, is tempting because a stale free looks like a harmless `-1` no-op. It is only harmless while the freed address stays unclaimed. The native registry rejects an address it no longer tracks, but once another thread allocates a fresh tracked object at that recycled address, the registry does track it again — and a stale free aimed at the old value would now find a live entry and destroy a different thread's object. The scenario is unlikely, but not unreachable: it needs a second thread inside its own FFI call, an allocator that hands back the exact address just freed, and that reuse to happen during the (narrow) window between the native drop and this free. But the window is real under concurrent use. The failure is a silent cross-thread corruption rather than a clean error, and the free is not needed in the first place on this branch. So where the value is known to be consumed, the free is skipped rather than issued and left to the registry to reject. The native error slot stays sticky: it holds whatever it last held until the next error overwrites it, and nothing clears it in between. Issuing an unneeded free would set an untracked-pointer error there that a later caller could mistake for the failure it actually asked about, so skipping the free keeps the slot free for the next real error. -So a taken-then-failed error means the value is already gone, and `_teardown(free_handle=False)` is exact: a `c2pa_free` there would be a guarded no-op that only dirties the sticky error slot and risks racing a recycled address. `_release_handle()` (a guarded free) is used only where ownership is unknown, such as an exception mid-call or the no-error fallthrough that no defined failure produces. There the guarded free is the right default: a real free if the handle is ours, a `-1` no-op if not. +`_release_handle()` (a guarded free) is reserved for the two branches where ownership is not known for certain: a Python exception raised before native reports anything, and a failure that leaves the error slot empty (which no defined native failure is expected to produce). In both, a guarded free is a good default, since it is a real free when the handle is still ours and a `-1` no-op when the native side already took it. -The native side is a prebuilt binary and not visible from this repo, so this two-outcome contract is the behavior the code is written against rather than something it can inspect. +A consuming C FFI function first removes the pointer from its registry, then reconstructs the owned value from it. `untrack_or_return!` runs ahead of `Box::from_raw` in `c2pa_c_ffi`. If the address is unknown or the wrong type, the untrack step fails before ownership is taken and sets an error whose prefix (`UntrackedPointer:` or `WrongPointerType:`) identifies it as a pre-consume rejection. Once the value has been reconstructed, a later failure simply drops it, the same as any owned value going out of scope. The Python side stays defensive (and as generic as possible) rather than assuming any exact behavior: it retains the handle when it recognizes one of those rejection prefixes, and where the outcome is unclear it falls back to the guarded free. A native side that behaved differently would degrade in one of two bounded ways: If it kept a pointer the Python side treated as consumed, nothing would free that pointer and it would leak. If it had already released a pointer the Python side then tried to free, the registry would not find the address and the free would return `-1` without touching memory. ### Adopting the handle before giving it away From 7ae2a20ba2dd40a382cf3afb415fd936eb77edc6 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Wed, 22 Jul 2026 09:52:08 -0700 Subject: [PATCH 31/67] fix: mermaid --- docs/native-resources-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index da5c289b..508ab8cd 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -294,7 +294,7 @@ sequenceDiagram participant N as Native lib C->>X: Context(settings, signer) - X->>B: with _NativeBuilder() (owns the builder; close() frees it on any failure) + X->>B: with _NativeBuilder() (owns the builder, close() frees it on any failure) X->>S: _ensure_valid_state() X->>X: copy signer._callback_cb to _signer_callback_cb Note right of X: Pin the callback first:
the Signer is about to be consumed From 2e03ded83ee25ba2763efc5da142dcdaef7d5d08 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:26:47 -0700 Subject: [PATCH 32/67] fix: Update docs to clarify double-free and such --- docs/native-resources-management.md | 74 ++++++++++++++++++++++++----- tests/test_unit_tests.py | 26 ++++++++++ 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index da5c289b..2035187f 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -80,6 +80,31 @@ Notes: > > Putting `ManagedResource` first in the declaration matters: the concrete `is_valid` implementation is found immediately during lookup, rather than hitting the abstract declaration on `ContextProvider` first. +## Python frees only what Python owns + +The C FFI is consistent about one thing that shapes this whole layer: some calls consume the pointer passed to them and hand back a replacement, because the native side may free and reallocate the underlying value. A pointer that went into a consuming call must never be freed by Python afterwards. Its address may already have been reallocated to a different object (address space is not infinite, so addresses get reused). + +Python owns and frees two kinds of things: the **single current native handle** for each object, and **Python-side resources it created itself** (stream wrappers, callbacks pinned so the native side can call back into them, caches). It swaps that one tracked handle to whatever a consuming call returns and, on the success path, never frees the value the call took. Beyond those owned resources it also carries bookkeeping it never frees (lifecycle state, the owning process ID, a borrowed reference to a caller-supplied `Context`), and it does not manage native reallocation itself: it swaps handles and, on the ambiguous failure paths, reads the native error tags to decide who still owns the pointer rather than assuming. + +Therefore, the managed resources have the following principles: + +- Each `ManagedResource` holds exactly one `_handle`. `_swap_handle()` replaces it with the pointer a consuming call returned and does not free the old value, since the native side took it (see [Consume-and-swap](#consume-and-swap)). +- `_teardown(free_handle=False)`, `_consume_no_replacement()`, and `_consume_into()` all close or advance the object without calling `c2pa_free`, because ownership moved to the native side. +- Only a few sites free a live handle. Two of them free a pointer this layer still provably owns: normal teardown (`_teardown(free_handle=True)`), and the create-then-validate path, which frees a freshly created pointer if activation fails. The third, `_release_handle()`, is a *guarded* free used only when ownership is genuinely unknown (a consuming call failed without setting an error, or a Python exception was raised before the native side reported anything): if the native side already took the pointer, its address is no longer in the registry and `c2pa_free` is a `-1` no-op, so the free touches no memory. No path frees a pointer known to have been consumed and reallocated (see [Why an ownership-taken failure does not free](#why-an-ownership-taken-failure-does-not-free)). +- `_release()` drops stream wrappers, callbacks, and caches before the native pointer is freed (see [Subclass-specific cleanup with `_release()`](#subclass-specific-cleanup-with-_release)). + +### Double-free risk mitigations + +Three distinct risks. Two have a mechanism in this layer; the third is the caller's to synchronize: + +| Hazard | Covered by | How | +| --- | --- | --- | +| Freeing a pointer a consuming call already took (single flow) | `_swap_handle` / `_teardown(free_handle=False)` triage | The consumed pointer is abandoned, never freed. The retained-vs-consumed decision reads the native error tag (`UntrackedPointer:` / `WrongPointerType:` mean not taken). | +| A forked child freeing a pointer its parent owns | PID stamp (`record_owner_pid` / `is_foreign_process`) | Cleanup in a process that did not allocate the pointer nulls the handle and marks `CLOSED` without freeing (see [Fork safety](#fork-safety)). | +| Two **threads** in one process racing frees on distinct objects, where the allocator recycles a just-freed address | Not covered here | `ManagedResource` has no lock and no thread stamping. The PID stamp cannot see it: sibling threads share a PID. Safety for genuinely shared handles must come from the caller's own synchronization or from the native registry, not this layer. | + +The PID stamp is fork-only: it compares process IDs, and two threads in the same process always match. Sharing one `ManagedResource` instance across threads without external synchronization is outside what this layer protects against. + ## Guarantees provided by ManagedResource `ManagedResource` provides the following guarantees, invariants must be maintained when subclassing the `ManagedResource` class in new implementation/new native resources handlers: @@ -145,15 +170,17 @@ Each transition has one method that performs it, and subclasses must go through | `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | | `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | | `_teardown(free_handle=False)` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | -| `_release_handle()` | ACTIVE to CLOSED | Frees the handle eagerly (via `_teardown(free_handle=True)`) and closes the object. Same post-state as the consumed teardown. | +| `_release_handle()` | ACTIVE to CLOSED | Frees the handle (guarded, via `_teardown(free_handle=True)`) and closes the object. Same post-state as the consumed teardown. | Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. +Two terms recur throughout this document. An **owned free** calls `c2pa_free` on a pointer this layer still provably holds: the normal `close()` / `__del__` path and the create-then-validate failure path both do this. A **guarded free** is the same call made when ownership is uncertain, which is what `_release_handle()` does: the native pointer registry tolerates being asked to free an address it no longer tracks, returning `-1` instead of crashing, so the free does not double-free a pointer the native side already took. That tolerance makes a guarded free safe to *issue*, but it is not free of consequence under concurrency — on a branch where the value is already known to be consumed, the layer skips the free rather than relying on the `-1`, because a stale free can race a recycled address (see [Why an ownership-taken failure does not free](#why-an-ownership-taken-failure-does-not-free)). + `_teardown(free_handle)` is the one method that performs the ACTIVE to CLOSED transition, and the boolean decides the only thing that varies between the two exit paths: whether the native pointer is freed. Both paths run `_release()`, set `CLOSED`, and null the handle. | `free_handle` | When | What it does with the pointer | | --- | --- | --- | -| `True` | We still own the pointer: normal `close()`, `__del__`, or a failure where ownership is unknown (`_release_handle()`). | Calls `c2pa_free` (guarded). | +| `True` | Either the pointer is still provably ours (normal `close()`, `__del__`) — an owned free — or ownership is unknown after a failure (`_release_handle()`) — a guarded free. | Calls `c2pa_free`. On the owned paths the pointer is really freed; on the unknown-ownership path the registry returns `-1` without touching memory if the native side already took it. | | `False` | The native side already took ownership: a consuming FFI call swallowed the pointer, or it passed to another object. | Frees nothing; the new owner does. A `c2pa_free` here would double-free (or hit the guarded `-1` no-op that dirties the error slot and risks racing a recycled address). | Every public method calls `_ensure_valid_state()` before doing any work, which raises `C2paError` unless the resource is ACTIVE with a non-null handle. @@ -184,7 +211,7 @@ Calling `.close()` directly is equivalent to exiting a `with` block. It is idemp ### Destructor fallback (`__del__`) -If neither of the above is used, `__del__` attempts to free the native pointer when Python garbage-collects the object. As described above, `__del__` timing is unpredictable and it may not run at all, so it is a safety net rather than a primary cleanup mechanism. +If neither the context manager nor an explicit `.close()` is used, `__del__` attempts to free the native pointer when Python garbage-collects the object. Per [Why `__del__` is not reliable enough](#why-__del__-is-not-reliable-enough), its timing is unpredictable and it may not run at all, so it is a safety net rather than a primary cleanup mechanism. ## Error handling during cleanup @@ -214,7 +241,7 @@ flowchart TD H -->|yes| FREE["_free_native_ptr()
logs on failure"] --> NULL["_handle = None"] --> DONE ``` -The `foreign process` branch is explained under [Fork safety](#fork-safety) below. +The `foreign process` branch is explained under [Fork safety](#fork-safety). ## Nesting resources @@ -234,7 +261,7 @@ with open("photo.jpg", "rb") as file: manifest = reader.json() ``` -The order matters because resources often depend on each other. In the example above, the `Reader` holds a native pointer that references the file's data through a `Stream` wrapper. If the file handle were closed first, the native library would still hold a pointer into the stream's read callbacks, and any subsequent access (including cleanup) could read freed memory or trigger a segfault. By closing the Reader first, the native pointer is freed while the underlying file is still open and valid. Python's `with` statement guarantees this ordering: resources listed later (or nested deeper) are torn down first. +The order matters because resources often depend on each other. In both examples, the `Reader` holds a native pointer that references the file's data through a `Stream` wrapper. If the file handle were closed first, the native library would still hold a pointer into the stream's read callbacks, and any subsequent access (including cleanup) could read freed memory or trigger a segfault. By closing the Reader first, the native pointer is freed while the underlying file is still open and valid. Python's `with` statement guarantees this ordering: resources listed later (or nested deeper) are torn down first. ## Reader lifecycle @@ -283,7 +310,7 @@ Some operations transfer a native pointer from one object to another. When this In the SDK this happens in one place: passing a `Signer` to a `Context`. The Context runs a short-lived native context builder, feeds the signer into it, builds the context, and activates the result. The builder itself is wrapped in `_NativeBuilder` (a small `ManagedResource`), so every failure inside the `with` block frees it through `close()` unless a consuming call already took it. There is no raw pointer held across the calls and no bespoke error handler. -The signer transfer goes through `_consume_no_replacement()`, which routes any failure to the shared triage (see [Consume-and-swap](#consume-and-swap)). That triage decides per error whether the signer was actually consumed: `set_signer` does *not* unconditionally take ownership. +The signer transfer goes through `_consume_no_replacement()`, which routes any failure to the shared triage (see [Why an ownership-taken failure does not free](#why-an-ownership-taken-failure-does-not-free)). That triage decides per error whether the signer was actually consumed: `set_signer` does *not* unconditionally take ownership. ```mermaid sequenceDiagram @@ -352,7 +379,7 @@ stateDiagram-v2 end note ``` -On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from a consumed teardown (`_teardown(free_handle=False)`), where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage below. +On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from a consumed teardown (`_teardown(free_handle=False)`), where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage in [`_consume_and_swap()`](#_consume_and_swap). ### `_consume_and_swap()` @@ -367,7 +394,7 @@ self._consume_and_swap( The call is passed as a lambda because the helper supplies the handle and, on success, replaces it via `_swap_handle()`. -The helper exists because a failed return can be ambiguous. The native functions run in phases: it validates the borrowed pointer, then takes ownership, then does the work. A failure in the first phase and a failure after the second come back to Python as the same value (a null pointer, or a non-zero status), but they leave ownership in opposite places. +The helper exists because a failed return can be ambiguous. The native functions run in phases: it validates the **borrowed pointer** (passed in without transferring ownership; the caller still owns it unless the callee explicitly takes it over), then takes ownership, then does the work. A failure in the first phase and a failure after the second come back to Python as the same value (a null pointer, or a non-zero status), but they leave ownership in opposite places. ```mermaid flowchart TD @@ -390,7 +417,7 @@ The two failure paths are indistinguishable from the return value alone. Only th | Any other error | Taken, then the operation failed | `_teardown(free_handle=False)`: the native side already dropped the value, so nothing is freed here. Resource goes `CLOSED`, error typed from the native message. | | No error at all | Unknown | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | -This error and ownership triage flow relies on the native error still being readable (and correctly being the last error encountered) after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it. +This error and ownership triage relies on the native error still being readable (and correctly being the last error encountered) after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it. Three consume helpers share this triage; they differ only in what the FFI call returns on success: @@ -400,13 +427,13 @@ Three consume helpers share this triage; they differ only in what the FFI call r | `_consume_no_replacement()` | a status code (`0` = ok) | `_teardown(free_handle=False)`, resource `CLOSED` | | `_consume_into()` | a *different* object's pointer | `_teardown(free_handle=False)`, the pointer returned for the caller to own | -`_consume_no_replacement()` is how a `Signer` is fed to a `Context` (`set_signer` returns a status code); `_consume_into()` is how that same `Context` build returns the new context pointer. A failure in any of the three is handled by the table above, so a pre-consume rejection retains the handle rather than assuming it was taken. +`_consume_no_replacement()` is how a `Signer` is fed to a `Context` (`set_signer` returns a status code); `_consume_into()` is how that same `Context` build returns the new context pointer. A failure in any of the three is handled by the same native-error triage, so a pre-consume rejection retains the handle rather than assuming it was taken. #### Why an ownership-taken failure does not free A consuming FFI call can fail. It may reject the borrowed pointer before taking it, or it may take ownership first and then, on a later failure, drop the value itself. -The native error message indicates which of the errors happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS` (`UntrackedPointer:` or `WrongPointerType:`), which means the handle was never taken and is retained. Any other error message means the native side may have taken ownership and already dropped the value. On top of those, a consuming call whose wrapper runs Python (such as a signing callback) can raise a Python exception before native reports anything at all, and that outcome is handled separately. +The native error message indicates which of the errors happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS` (`UntrackedPointer:` or `WrongPointerType:`), which means the handle was never taken and is retained. Any other error message means the native side may have taken ownership and already dropped the value. On top of those, preparing the call's own arguments can fail in Python before the native function ever runs (for example, encoding a bad value or a ctypes marshalling error other than `ArgumentError`), and that outcome is handled separately. The two settled branches each take the exact action their ownership implies. A pre-consume rejection (an error prefixed `UntrackedPointer:` or `WrongPointerType:`) means the handle is still the caller's, so it is retained and freed later by normal cleanup. Any other native error means the value is already gone, so `_teardown(free_handle=False)` runs the Python-side cleanup without freeing anything. @@ -414,6 +441,8 @@ Always calling the guarded free instead, even where the value is known to be gon `_release_handle()` (a guarded free) is reserved for the two branches where ownership is not known for certain: a Python exception raised before native reports anything, and a failure that leaves the error slot empty (which no defined native failure is expected to produce). In both, a guarded free is a good default, since it is a real free when the handle is still ours and a `-1` no-op when the native side already took it. +None of this is protected by a lock on the Python side: `ManagedResource` has no thread-safety mechanism of its own, and the retained-vs-consumed guarantee comes entirely from the native pointer registry and its thread-local error slot. As noted under [Which double-free risks this layer guards](#which-double-free-risks-this-layer-guards-and-which-it-leaves-to-the-caller), sharing one instance across threads without external synchronization is the caller's responsibility. This is a different hazard from [Fork safety](#fork-safety), which concerns a forked child process, not a thread within the same process. + A consuming C FFI function first removes the pointer from its registry, then reconstructs the owned value from it. `untrack_or_return!` runs ahead of `Box::from_raw` in `c2pa_c_ffi`. If the address is unknown or the wrong type, the untrack step fails before ownership is taken and sets an error whose prefix (`UntrackedPointer:` or `WrongPointerType:`) identifies it as a pre-consume rejection. Once the value has been reconstructed, a later failure simply drops it, the same as any owned value going out of scope. The Python side stays defensive (and as generic as possible) rather than assuming any exact behavior: it retains the handle when it recognizes one of those rejection prefixes, and where the outcome is unclear it falls back to the guarded free. A native side that behaved differently would degrade in one of two bounded ways: If it kept a pointer the Python side treated as consumed, nothing would free that pointer and it would leak. If it had already released a pointer the Python side then tried to free, the registry would not find the address and the free would return `-1` without touching memory. ### Adopting the handle before giving it away @@ -500,6 +529,25 @@ The reason is that ownership runs in the opposite direction. A `Reader` or `Buil `Stream` tracks its own state with `_closed` and `_initialized` flags rather than `LifecycleState`, but it supports the same three cleanup paths: context manager, explicit `.close()`, and `__del__` fallback. +## Which method to use when? + +`_create_and_activate`, `_consume_and_swap`, `_consume_no_replacement`, +`_consume_into`, `_release_handle`, and `_wrap_native_handle` each apply to a +different situation when writing a new subclass: + +| Situation | Call this | +| --- | --- | +| Creating a brand-new native object from an FFI constructor | `_create_and_activate(ffi_call, error_message)` | +| An FFI call consumes the current handle and returns a replacement for the same object | `_consume_and_swap(ffi_call, error_message)` | +| An FFI call consumes the current handle to configure or feed another object, returning only a status code | `_consume_no_replacement(ffi_call, error_message)` | +| An FFI call consumes the current handle and returns a *different* object's pointer, for that object to own | `_consume_into(ffi_call, error_message)` | +| A call fails and it is unclear whether native took the handle first (a Python exception before native reported anything, or an empty error slot) | `_release_handle()` | +| A Python instance needs to wrap a handle a native call already returned, without creating a new one | `_wrap_native_handle(handle)` (classmethod) | +| Ordinary teardown (`close()`, `__del__`) | Neither: these already route through `_cleanup_resources()` and `_teardown()`. Nothing outside `ManagedResource` itself calls `_teardown()` directly. | + +`_activate()` and `_swap_handle()` are two low-level primitives this +situation table builds on. + ## Implementing a subclass of `ManagedResource` To wrap a new native resource, inherit from `ManagedResource` and follow these rules: @@ -566,11 +614,11 @@ class NativeResource(ManagedResource): - Assigning `self._handle` or `self._lifecycle_state` directly bypasses the checks that make the lifecycle safe. `_activate()` refuses a null handle and refuses to run on an already-active object; `_swap_handle()` requires the resource to be active and the replacement non-null. Direct assignment gives up both, and the resulting bugs (an ACTIVE object with a null handle, or a silently discarded pointer) surface far from their cause. -- A `_release()` that raises has its exception silently swallowed by `_cleanup_resources()`, visible only in the logs. A small lifecycle for managed resources lets `_release()` check whether they need releasing; the actual release call wrapped in try/except is a fallback for unexpected failures. +- A `_release()` that raises has its exception silently swallowed by `_cleanup_resources()`, visible only in the logs. A small lifecycle for managed resources would let `_release()` check whether they need releasing; the actual release call wrapped in try/except is a fallback for unexpected failures. - `_release()` can be called more than once (via `close()` then `__del__`, or multiple `close()` calls), so it must handle being called on an already-cleaned-up object. Setting attributes to `None` after closing them is the standard pattern. -- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the unknown-ownership failure paths can free eagerly without risking a double-free. A manual free is still wrong — the lifecycle owns the pointer and bypassing it defeats the state checks. +- Calling `c2pa_free` directly is not recommended. `ManagedResource` handles this. A redundant free of an already-released pointer is not a crash: the native pointer registry rejects an untracked address without touching memory and returns `-1`. `ManagedResource` relies on this guard so the unknown-ownership failure paths can issue a guarded free without risking a double-free. A manual free is still wrong — the lifecycle owns the pointer and bypassing it defeats the state checks. - When a subclass inherits from both `ManagedResource` and an ABC like `ContextProvider`, and both define a property with the same name (e.g. `is_valid`), Python resolves it using the MRO. The parent listed first in the class definition wins. With the ABC listed first, Python finds the abstract property before the concrete one and raises `TypeError: Can't instantiate abstract class`. The class with the concrete implementation therefore comes first (e.g. `class Context(ManagedResource, ContextProvider)`, not `class Context(ContextProvider, ManagedResource)`). diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 4cc05aa1..ef942554 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -3806,6 +3806,32 @@ def test_builder_initialization_failure_states(self): with self.assertRaises(Exception) as context: builder = Builder(circular_obj) + def test_construction_failure_before_native_call_is_collectable(self): + """Builder(None) fails encoding the manifest before any FFI call. + A half-built instance is UNINITIALIZED with no native handle. + """ + captured = [] + real_init_attrs = Builder._init_attrs + + def recording_init_attrs(self): + real_init_attrs(self) + captured.append(self) + + Builder._init_attrs = recording_init_attrs + try: + with self.assertRaises(Error): + Builder(None) + finally: + Builder._init_attrs = real_init_attrs + + self.assertEqual(len(captured), 1, + "_init_attrs did not run during the failed construction") + instance = captured[0] + self.assertEqual(instance._lifecycle_state, LifecycleState.UNINITIALIZED) + self.assertIsNone(instance._handle) + + instance._release() + def test_builder_state_transitions(self): """Test Builder state transitions during lifecycle.""" builder = Builder(self.manifestDefinition) From e58e113982e1a39fea3998de98f01b29f2c23fe9 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:24:06 -0700 Subject: [PATCH 33/67] fix: Review comments --- docs/native-resources-management.md | 12 ++++++++++++ src/c2pa/c2pa.py | 23 +++++++++++++++-------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 40af9dcb..1127093b 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -16,6 +16,18 @@ A new wrapper around a native resource inherits from `ManagedResource` and follows the documented lifecycle rules. +## Definitions + +A **native pointer** is an address that says where a piece of memory lives. The C2PA SDK is a Python wrapper around a Rust library (the "native" library), and that library allocates memory Python cannot see. When the SDK creates a `Reader`, `Builder`, `Signer`, `Context`, or `Settings`, the Python object holds a native pointer to memory that the native library allocated and manages. Python's garbage collector tracks the Python object but knows nothing about the native memory behind the pointer, so it cannot free it. + +The **native side** of the Rust library is reached through its C FFI. + +**Ownership** answers one question: who is responsible for freeing a piece of native memory. Native memory has to be freed exactly once. The owner is whoever must free it. If nobody frees it, the memory leaks. If two owners each free it, the same memory is freed twice, which corrupts the allocator and can crash the process. So exactly one side owns each pointer at any moment, and that side frees it. + +**Taking ownership** or **transferring ownership** means that responsibility moves from one holder to another. When Python hands a pointer to the native side and the native side takes ownership, the old holder must stop trying to free it, or the pointer gets freed twice. The [Ownership transfer](#ownership-transfer) section covers how the Python SDK handles this. + +A pointer is **consumed** when a native call takes ownership of the pointer passed to it, often returning a replacement pointer. Once a pointer is consumed, Python must not free it again, since the native side now owns it. + ## Why is native resources management needed? ### Native pointers in a Python wrapper diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 4e334698..97c5c335 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -376,6 +376,15 @@ def _create_and_activate(self, ffi_call, error_message, *, """Obtain a fresh native pointer, validate it, and take ownership. On any failure before ownership transfers, the pointer is freed and the error re-raised. + + Args: + ffi_call: Zero-arg callable returning a fresh native pointer. + error_message: Message for the C2paError raised on failure. + check: Predicate marking a result invalid + (default: a falsy pointer). + + Raises: + C2paError: If the pointer fails validation; it is freed first. """ ptr = ffi_call() try: @@ -1669,9 +1678,9 @@ class _NativeBuilder(ManagedResource): def __init__(self): super().__init__() - ptr = _lib.c2pa_context_builder_new() - _check_ffi_operation_result(ptr, "Failed to create ContextBuilder") - self._activate(ptr) + self._create_and_activate( + _lib.c2pa_context_builder_new, + "Failed to create ContextBuilder") def __init__( self, @@ -1694,11 +1703,9 @@ def __init__( if settings is None and signer is None: # Simple default context - context_ptr = _lib.c2pa_context_new() - _check_ffi_operation_result( - context_ptr, "Failed to create Context" - ) - self._activate(context_ptr) + self._create_and_activate( + _lib.c2pa_context_new, + "Failed to create Context") else: # Any failure inside the with frees the builder via close(); # a successful build consumes it, so close() is then a no-op. From b998747c418568bda19fbda24c751e53fd2fd9c8 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:43:56 -0700 Subject: [PATCH 34/67] fix: Docs typo --- docs/native-resources-management.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 1127093b..52a10981 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -22,7 +22,9 @@ A **native pointer** is an address that says where a piece of memory lives. The The **native side** of the Rust library is reached through its C FFI. -**Ownership** answers one question: who is responsible for freeing a piece of native memory. Native memory has to be freed exactly once. The owner is whoever must free it. If nobody frees it, the memory leaks. If two owners each free it, the same memory is freed twice, which corrupts the allocator and can crash the process. So exactly one side owns each pointer at any moment, and that side frees it. +A **handle** is a single native pointer a `ManagedResource` object holds and manages, stored in its `_handle` attribute. Each object owns one handle at a time. The lifecycle machinery is mostly about tracking that one handle: creating it, swapping it, and freeing it. + +**Ownership** answers one question: who is responsible for freeing a piece of native memory. Native memory has to be freed (exactly once). The owner is whoever must free it. If nobody frees it, the memory leaks. If two owners each free it, the same memory is freed twice, which corrupts the allocator and can crash the process. So exactly one side owns each pointer at any moment, and that side frees it. **Taking ownership** or **transferring ownership** means that responsibility moves from one holder to another. When Python hands a pointer to the native side and the native side takes ownership, the old holder must stop trying to free it, or the pointer gets freed twice. The [Ownership transfer](#ownership-transfer) section covers how the Python SDK handles this. From 31c0a43f28dc4b10f1453d304d17fcd60dd5aebe Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:18:06 -0700 Subject: [PATCH 35/67] Update read.py --- examples/read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/read.py b/examples/read.py index b2ef9fc6..e4b718a9 100644 --- a/examples/read.py +++ b/examples/read.py @@ -36,7 +36,7 @@ def read_c2pa_data(media_path: str): # All objects using this context will have trust configured. with c2pa.Context(settings) as context: with c2pa.Reader(media_path, context=context) as reader: - print(reader.crjson()) + print(reader.detailed_json()) except Exception as e: print(f"Error reading C2PA data from {media_path}: {e}") sys.exit(1) From 4c4bd1e7fa6495554a180c2940e7c5502801b189 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:18:33 -0700 Subject: [PATCH 36/67] Change print statement from crjson to json --- examples/sign.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sign.py b/examples/sign.py index 09133b35..e6c14859 100644 --- a/examples/sign.py +++ b/examples/sign.py @@ -110,6 +110,6 @@ def callback_signer_es256(data: bytes) -> bytes: # The validation state will depend on loaded trust settings. # Without loaded trust settings, # the manifest validation_state will be "Invalid". - print(reader.crjson()) + print(reader.json()) print("\nExample completed successfully!") From 88d6e435e1fbfbd29ea4609f4214aca488a4acf4 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:19:02 -0700 Subject: [PATCH 37/67] Update print statements to use reader.json() --- examples/sign_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/sign_info.py b/examples/sign_info.py index 3735ad0b..6b256647 100644 --- a/examples/sign_info.py +++ b/examples/sign_info.py @@ -40,7 +40,7 @@ print("\nReading existing C2PA metadata:") with open(fixtures_dir + "C.jpg", "rb") as file: with c2pa.Reader("image/jpeg", file) as reader: - print(reader.crjson()) + print(reader.json()) # Create a signer from certificate and key files with open(fixtures_dir + "es256_certs.pem", "rb") as cert_file: @@ -103,7 +103,7 @@ # The validation state will depend on loaded trust settings. # Without loaded trust settings, # the manifest validation_state will be "Invalid". - print(reader.crjson()) + print(reader.json()) print("\nExample completed successfully!") From e8f0c77cb4a0f93aba7bfb654f97f59ed3d85c0f Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:41:26 -0700 Subject: [PATCH 38/67] fix: Add tests to verify TSA URL use --- src/c2pa/c2pa.py | 93 ++++++++++++------- tests/test_unit_tests.py | 146 ++++++++++++++++++++++++++++++ tests/test_unit_tests_threaded.py | 5 +- 3 files changed, 207 insertions(+), 37 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index bf02703d..851348f8 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -462,6 +462,13 @@ def _raise_consume_failure(self, error_message): sticky and thread-local and the SDK does not clear it before the call, so this trusts that the failing native path set its own error. + That ordering is required: + c2pa_free on a handle the registry no longer tracks returns -1 and + overwrites the slot with its own "Other: UntrackedPointer: 0x..." + message. Freeing first would therefore replace the real failure + with another one and, because that substitute carries a pre-consume + tag, invert the retain/consume decision made below. + Args: error_message: Format string with one placeholder, used when the native layer offers no error of its own. @@ -725,7 +732,8 @@ def __init__(self, alg, sign_cert, private_key, ta_url): (will be converted accordingly to bytes for native library use) sign_cert: The signing certificate as a string private_key: The private key as a string - ta_url: The timestamp authority URL as bytes + ta_url: The timestamp authority URL as bytes, or None for no + timestamp authority. """ if sign_cert is None: @@ -754,8 +762,11 @@ def __init__(self, alg, sign_cert, private_key, ta_url): ) # Handle ta_url parameter: - # allow string or bytes, convert string to bytes as needed - if isinstance(ta_url, str): + # allow None, string or bytes, convert string to bytes as needed + if ta_url is None: + # NULL is how the native side spells "no timestamp authority". + pass + elif isinstance(ta_url, str): # String to bytes, as requested by native lib ta_url = ta_url.encode('utf-8') elif isinstance(ta_url, bytes): @@ -763,7 +774,7 @@ def __init__(self, alg, sign_cert, private_key, ta_url): pass else: raise TypeError( - f"ta_url must be string or bytes, got {type(ta_url)}" + f"ta_url must be None, string or bytes, got {type(ta_url)}" ) # Call parent constructor with processed values @@ -1138,7 +1149,6 @@ class _C2paVerify(C2paError): class _C2paUntrackedPointer(C2paError): """Exception raised when the native layer does not recognize a pointer. - Raised when a consume-and-return call rejects the handle it was given before taking ownership of it, so the caller still owns that handle. """ @@ -1147,7 +1157,6 @@ class _C2paUntrackedPointer(C2paError): class _C2paWrongPointerType(C2paError): """Exception raised when a pointer is tracked under a different type. - Like _C2paUntrackedPointer, this is rejected before ownership transfer, so the caller still owns the handle it passed in. """ @@ -1454,8 +1463,6 @@ def load_settings(settings: Union[str, dict], format: str = "json") -> None: "Error loading settings", check=lambda r: r != 0) - return result - def _get_mime_type_from_path(path: Union[str, Path]) -> str: """Attempt to guess the MIME type from a file path's extension. @@ -1611,7 +1618,8 @@ def update( @property def _c_settings(self): - """Expose the raw pointer for Context to consume.""" + """Expose the raw pointer for c2pa_context_builder_set_settings. + """ self._ensure_valid_state() return self._handle @@ -2530,28 +2538,26 @@ def _create_reader(self, format_bytes, stream_obj, """ format_arg = _format_ffi_arg(format_bytes) if manifest_data is None: - reader_ptr = _lib.c2pa_reader_from_stream( - format_arg, stream_obj._stream) + def create(): + return _lib.c2pa_reader_from_stream( + format_arg, stream_obj._stream) else: if not isinstance(manifest_data, bytes): raise TypeError(Reader._ERROR_MESSAGES['manifest_error']) manifest_array = ( ctypes.c_ubyte * len(manifest_data)).from_buffer_copy(manifest_data) - reader_ptr = ( - _lib.c2pa_reader_from_manifest_data_and_stream( + + def create(): + return _lib.c2pa_reader_from_manifest_data_and_stream( format_arg, stream_obj._stream, manifest_array, len(manifest_data), ) - ) - - _check_ffi_operation_result( - reader_ptr, - Reader._ERROR_MESSAGES['reader_error']) - self._activate(reader_ptr) + self._create_and_activate( + create, Reader._ERROR_MESSAGES['reader_error']) def _init_from_file(self, path, format_bytes, manifest_data=None): @@ -2722,7 +2728,11 @@ def with_fragment(self, format: Optional[str], stream, This reader instance, for method chaining. Raises: - C2paError: If there was an error processing the fragment + C2paError: If there was an error processing the fragment. + On failure the native call may already have consumed the + underlying object, in which case this Reader is closed and + cannot be retried: create a new one instead of reusing this + instance. """ self._ensure_valid_state() @@ -3014,7 +3024,12 @@ def from_info(cls, signer_info: C2paSignerInfo) -> 'Signer': _check_ffi_operation_result( signer_ptr, "Failed to create signer from configured signer_info") - return cls(signer_ptr) + try: + return cls(signer_ptr) + except Exception: + # No instance took ownership, so the handle is still ours. + ManagedResource._free_native_ptr(signer_ptr) + raise @classmethod def from_callback( @@ -3141,13 +3156,18 @@ def wrapped_callback( _check_ffi_operation_result(signer_ptr, "Failed to create signer") - # Create and return the signer instance with the callback - signer_instance = cls(signer_ptr) + try: + # Create and return the signer instance with the callback + signer_instance = cls(signer_ptr) - # Keep callback alive on the object to prevent gc of the callback - # So the callback will live as long as the signer leaves, - # and there is a 1:1 relationship between signer and callback. - signer_instance._callback_cb = callback_cb + # Keep callback alive on the object to prevent gc of the callback. + # So the callback will live as long as the signer leaves, + # and there is a 1:1 relationship between signer and callback. + signer_instance._callback_cb = callback_cb + except Exception: + # No instance took ownership, so the handle is still ours. + ManagedResource._free_native_ptr(signer_ptr) + raise return signer_instance @@ -3355,14 +3375,10 @@ def __init__( if context is not None: self._init_from_context(context, json_str) else: - builder_ptr = _lib.c2pa_builder_from_json(json_str) - - _check_ffi_operation_result( - builder_ptr, + self._create_and_activate( + lambda: _lib.c2pa_builder_from_json(json_str), Builder._ERROR_MESSAGES['builder_error']) - self._activate(builder_ptr) - def _init_from_context(self, context, json_str): """Initialize Builder from a ContextProvider. @@ -3657,7 +3673,10 @@ def with_archive(self, stream: Any) -> 'Builder': This builder instance, for method chaining. Raises: - C2paError: If there was an error loading the archive + C2paError: If there was an error loading the archive. On failure + the native call may already have consumed the underlying + object, in which case this Builder is closed and cannot be + retried: create a new one instead of reusing this instance. """ self._ensure_valid_state() @@ -3780,7 +3799,11 @@ def _sign_common( """ source_stream = Stream(source) try: - if dest: + # Identity check, not truthiness: a caller-supplied destination + # that is falsy while empty (any stream-like object defining + # __bool__/__len__) would otherwise be silently swapped for the + # internal buffer and never receive the signed asset. + if dest is not None: dest_stream = Stream(dest) else: mem_buffer = io.BytesIO() diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 60740c9e..837e6d66 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -1611,6 +1611,139 @@ def test_reserve_size(self): signer = Signer.from_info(signer_info) signer.reserve_size() + def _local_signer(self): + """An es256 signer with no timestamp authority (keeps signing local). + + ta_url=None maps to a NULL pointer, which is how the native side + spells "no TSA". An empty string is not equivalent: it is passed + through as a real, empty URL and signing then fails. + """ + signer_info = C2paSignerInfo( + alg=b"es256", + sign_cert=self.certs, + private_key=self.key, + ta_url=None, + ) + signer = Signer.from_info(signer_info) + self.addCleanup(signer.close) + return signer + + def _active_signature_info(self, signed_bytes): + """signature_info of the active manifest in a signed asset.""" + signed_bytes.seek(0) + reader = Reader("image/jpeg", signed_bytes) + try: + manifest_store = json.loads(reader.json()) + finally: + reader.close() + active = manifest_store["manifests"][ + manifest_store["active_manifest"]] + return active.get("signature_info", {}) + + def test_signer_with_none_ta_url_signs_without_timestamp(self): + """ta_url=None must reach native as NULL, meaning "no TSA". + """ + builder = Builder(self.manifestDefinitionV2) + dest = io.BytesIO() + with open(self.testPath, "rb") as source: + builder.sign(self._local_signer(), "image/jpeg", source, dest) + + signature_info = self._active_signature_info(dest) + self.assertTrue(signature_info, "expected a signed active manifest") + self.assertNotIn( + "time", signature_info, + "no timestamp authority was configured, so the signature " + "must not carry a timestamp") + + def _sign_with_ta_url(self, ta_url): + """Sign with ta_url as the only variable. Returns the dest buffer.""" + signer_info = C2paSignerInfo( + alg=b"es256", + sign_cert=self.certs, + private_key=self.key, + ta_url=ta_url, + ) + signer = Signer.from_info(signer_info) + self.addCleanup(signer.close) + + builder = Builder(self.manifestDefinitionV2) + dest = io.BytesIO() + with open(self.testPath, "rb") as source: + builder.sign(signer, "image/jpeg", source, dest) + return dest + + def test_signer_with_empty_ta_url_is_not_a_no_tsa_path(self): + """An empty ta_url is a real (empty) URL, not "no TSA". + Only None means "no TSA". + """ + with self.assertRaises(Error): + self._sign_with_ta_url(b"") + + def test_empty_ta_url_is_rejected_as_a_malformed_url(self): + """b"" fails as a URL, which is why it cannot mean "no TSA". + """ + malformed = [ + (b"", "empty"), + (b" ", "whitespace"), + (b"not-a-url", "no scheme or host"), + ] + for value, why in malformed: + with self.subTest(ta_url=value, reason=why): + with self.assertRaises(Error): + self._sign_with_ta_url(value) + + # The control: same signer, same asset, only ta_url differs. + signature_info = self._active_signature_info( + self._sign_with_ta_url(None)) + self.assertNotIn("time", signature_info) + + def test_signer_with_ta_url_signs_with_timestamp(self): + """Control for the None case: a real TSA does add a timestamp. + """ + builder = Builder(self.manifestDefinitionV2) + dest = io.BytesIO() + with open(self.testPath, "rb") as source: + builder.sign(self.signer, "image/jpeg", source, dest) + + signature_info = self._active_signature_info(dest) + self.assertIn( + "time", signature_info, + "a timestamp authority was configured, so the signature " + "must carry a timestamp") + self.assertTrue(signature_info["time"]) + + def test_none_and_empty_ta_url_reach_native_differently(self): + """None must marshal to NULL, b"" to a real (empty) string. + + The native side reads a NULL ta_url as "no timestamp authority" + and any non-NULL pointer as a URL to use, including one pointing at "". + """ + def struct_for(ta_url): + return C2paSignerInfo( + alg=b"es256", + sign_cert=self.certs, + private_key=self.key, + ta_url=ta_url, + ) + + # ctypes surfaces a NULL c_char_p as None and an empty one as b"". + self.assertIsNone(struct_for(None).ta_url) + self.assertEqual(struct_for(b"").ta_url, b"") + # A str is encoded rather than passed through untouched. + self.assertEqual( + struct_for("http://timestamp.digicert.com").ta_url, + b"http://timestamp.digicert.com") + + def test_signer_rejects_non_string_ta_url(self): + """Only None, str and bytes are accepted.""" + with self.assertRaises(TypeError): + C2paSignerInfo( + alg=b"es256", + sign_cert=self.certs, + private_key=self.key, + ta_url=1234, + ) + def test_signer_creation_error_alg(self): signer_info = C2paSignerInfo( alg=b"not-an-alg", @@ -9291,6 +9424,19 @@ def test_unmapped_tag_falls_back_to_base_error(self): # Base class only: no subclass should claim an unknown tag. self.assertIs(type(ctx.exception), Error) + def test_pre_consume_tag_match_is_substring_not_prefix(self): + """The tags arrive mid-string, so the match must stay a substring one. + + Guards the triage in _raise_consume_failure against being "cleaned up" + into error.startswith(tag), which would match nothing and silently + turn every retained handle into a consumed one. + """ + wire_error = "Other: UntrackedPointer: 0xdeadb000" + tags = ManagedResource._PRE_CONSUME_ERROR_TAGS + + self.assertTrue(any(tag in wire_error for tag in tags)) + self.assertFalse(any(wire_error.startswith(tag) for tag in tags)) + def test_check_ffi_operation_result_raises_with_native_message(self): self._set_native_error("Io: disk exploded") with self.assertRaises(Error) as ctx: diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index e7d49b16..d0d0b2c1 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -12,6 +12,7 @@ # each license. import ctypes +import gc import os import io import json @@ -2921,6 +2922,8 @@ class TestManagedResourceCrossThread(unittest.TestCase): """ def setUp(self): + # Flush pending finalizers through the real free first. + gc.collect() self.freed = [] self._real_free = ManagedResource._free_native_ptr ManagedResource._free_native_ptr = staticmethod(self.freed.append) @@ -2971,8 +2974,6 @@ def create(index): self.assertEqual(counts, expected) def test_third_thread_gc_of_dropped_reference_frees_exactly_once(self): - import gc - def make_and_drop(index): res = _ConcreteResource() res._activate(0x30000 + index) From f511414d7a6d21959da12527daf546987cad2113 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:31:25 -0700 Subject: [PATCH 39/67] fix: Initial resolver bindings --- examples/README.md | 49 +++ examples/http_resolver_cache.py | 236 ++++++++++++++ examples/http_resolver_debug.py | 158 +++++++++ src/c2pa/__init__.py | 4 + src/c2pa/c2pa.py | 367 ++++++++++++++++++++- tests/test_http_resolver.py | 553 ++++++++++++++++++++++++++++++++ 6 files changed, 1363 insertions(+), 4 deletions(-) create mode 100644 examples/http_resolver_cache.py create mode 100644 examples/http_resolver_debug.py create mode 100644 tests/test_http_resolver.py diff --git a/examples/README.md b/examples/README.md index 191e88f4..e9ef242a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -68,6 +68,35 @@ except Exception as err: sys.exit(err) ``` +## Using a custom HTTP resolver + +A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`: remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG `did:web` resolution. Use it to add authentication headers, cache responses, log traffic, or serve responses from memory in tests. + +A resolver is either an object with a `resolve(request)` method or a plain callable. It receives an `HttpRequest` and returns an `HttpResponse`: + +```py +class MyResolver: + def resolve(self, request): + # request.url, request.method, request.headers, request.body + return c2pa.HttpResponse(200, b"...") + +context = c2pa.Context.builder().with_resolver(MyResolver()).build() +reader = c2pa.Reader("image/jpeg", stream, context=context) +``` + +Raising from the resolver marks the request as a hard failure; returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. + +Two things are worth knowing before you write one: + +- A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. +- The SDK does not follow redirects. Delegating to `urllib.request` gives you redirect handling for free. + +The [`examples/http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_debug.py) script logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. It runs one read flow and one signing flow. + +The [`examples/http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_cache.py) script adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. It reads the same asset twice to show a cache hit, then adds the same remote-manifest ingredient three times while signing, which produces one network fetch and two cache hits. + +Both scripts use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so **they need internet access**. If the fetch fails they print a hint instead of a traceback. + ## Running the examples To run the examples, make sure you have the c2pa-python package installed (`pip install c2pa-python`) and you're in the root directory of the project. We recommend working using virtual environments (venv). Then run the examples as shown below. @@ -100,6 +129,26 @@ In this example, `SignerInfo` creates a `Signer` object that signs the manifest. python examples/sign_info.py ``` +### Run the debugging HTTP resolver example + +This example logs every HTTP request the SDK makes. It needs internet access. + +```bash +python examples/http_resolver_debug.py +``` + +Expected output: one `GET` of the Adobe manifest URL with a `-> 200` for the read flow, another `GET` when the remote-manifest ingredient is added while signing, and no HTTP at all when the signed result is read back (its manifest is embedded). + +### Run the caching HTTP resolver example + +This example caches responses and retries throttled requests. It needs internet access. + +```bash +python examples/http_resolver_cache.py +``` + +Expected output: `MISS` then `HIT` for the two reads, then `1 miss` and `2 hits` for the three ingredient additions during signing. It ends with a resolver that always answers 500, showing that a final failure surfaces as a typed `C2paError` rather than a crash. + ## Backend application example [c2pa-python-example](https://github.com/contentauth/c2pa-python-example) is an example of a simple application that accepts an uploaded JPEG image file, attaches a C2PA manifest, and signs it using a certificate. The app uses the CAI Python library and the Flask Python framework to implement a back-end REST endpoint; it does not have an HTML front-end, so you have to use something like curl to access it. This example is a development setup and should not be deployed as-is to a production environment. diff --git a/examples/http_resolver_cache.py b/examples/http_resolver_cache.py new file mode 100644 index 00000000..e354d828 --- /dev/null +++ b/examples/http_resolver_cache.py @@ -0,0 +1,236 @@ +import collections +import os +import sys +import threading +import time +import urllib.error +import urllib.request + +import c2pa + +# This example shows a custom HTTP resolver that caches responses and retries +# throttled requests, using nothing but the standard library. +# +# The SDK may call a resolver from worker threads, so everything here is +# thread-safe. +# +# This example needs internet access: tests/fixtures/cloud.jpg carries no +# embedded manifest, only a remote one that has to be fetched. + +fixtures_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "tests", "fixtures") + os.sep +output_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "output") + os.sep + + +class TtlLruCache: + """A small LRU cache whose entries also expire after a TTL.""" + + def __init__(self, max_items=100, ttl_seconds=120.0): + self._max_items = int(max_items) + self._ttl = float(ttl_seconds) + self._entries = collections.OrderedDict() # key -> (expiry, value) + self._lock = threading.Lock() + self.hits = 0 + self.misses = 0 + + def get(self, key): + with self._lock: + entry = self._entries.get(key) + # time.monotonic, not time.time: immune to wall-clock jumps. + if entry is not None and entry[0] > time.monotonic(): + self._entries.move_to_end(key) + self.hits += 1 + return entry[1] + if entry is not None: + del self._entries[key] + self.misses += 1 + return None + + def put(self, key, value): + with self._lock: + if key in self._entries: + self._entries.move_to_end(key) + self._entries[key] = (time.monotonic() + self._ttl, value) + while len(self._entries) > self._max_items: + self._entries.popitem(last=False) + + +class CachingHttpResolver: + """An HTTP resolver with a response cache and bounded retries. + + Caching policy: only GET requests answered with 200 are cached. POSTs + (timestamp requests) and error responses never are. + + Retry policy: 429 and 503 are retried up to max_retries times, honoring + a capped Retry-After when the header is present and otherwise backing + off exponentially. Any other status is final and is passed through to + the SDK. Transport errors raise, which marks a hard failure. + """ + + def __init__(self, cache=None, timeout=10.0, max_retries=3, + backoff_seconds=0.5, max_retry_after=10.0): + self.cache = cache if cache is not None else TtlLruCache() + self._timeout = timeout + self._max_retries = int(max_retries) + self._backoff = float(backoff_seconds) + self._max_retry_after = float(max_retry_after) + + def resolve(self, request): + cacheable = request.method.upper() == "GET" + if cacheable: + cached = self.cache.get(request.url) + if cached is not None: + print(f"[cache] HIT {request.url}", flush=True) + return c2pa.HttpResponse(cached[0], cached[1]) + print(f"[cache] MISS {request.url}", flush=True) + + status, body = self._fetch_with_retries(request) + if cacheable and status == 200: + self.cache.put(request.url, (status, body)) + return c2pa.HttpResponse(status, body) + + def _fetch_with_retries(self, request): + data = request.body or None + for attempt in range(self._max_retries + 1): + req = urllib.request.Request( + request.url, + data=data, + method=request.method, + headers=request.headers) + try: + with urllib.request.urlopen( + req, timeout=self._timeout) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + retryable = e.code in (429, 503) + if not retryable or attempt == self._max_retries: + # Final failure: pass the status back so the SDK can + # report it as a typed error. Not cached. + return e.code, e.read() + delay = self._retry_delay(e, attempt) + print(f"[http] {e.code}, retrying in {delay:.1f}s", + flush=True) + time.sleep(delay) + raise RuntimeError("unreachable") + + def _retry_delay(self, error, attempt): + retry_after = error.headers.get("Retry-After") + if retry_after: + try: + return min(float(retry_after), self._max_retry_after) + except ValueError: + pass + return self._backoff * (2 ** attempt) + + +def read_with_cache(): + """Read the same remote-manifest asset twice: one fetch, one cache hit.""" + print("\n--- Reading cloud.jpg twice through one cache ---") + + resolver = CachingHttpResolver() + with c2pa.Context.builder().with_resolver(resolver).build() as context: + for round_number in (1, 2): + with open(fixtures_dir + "cloud.jpg", "rb") as f: + with c2pa.Reader("image/jpeg", f, context=context) as reader: + print(f" read {round_number}: " + f"{reader.get_validation_state()}") + + print(f"cache hits={resolver.cache.hits} misses={resolver.cache.misses}") + + +def sign_with_cache(): + """Add the same remote-manifest ingredient repeatedly, hitting the cache. + + Each add re-reads the ingredient's remote manifest through the context + resolver, so three adds mean one network fetch and two cache hits. + """ + print("\n--- Signing with the same remote ingredient added 3 times ---") + + with open(fixtures_dir + "es256_certs.pem", "rb") as f: + certs = f.read() + with open(fixtures_dir + "es256_private.key", "rb") as f: + key = f.read() + + # ta_url is None, meaning "no timestamp authority". An empty string is + # treated as a URL and fails signing. + signer_info = c2pa.C2paSignerInfo( + alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) + + manifest_definition = { + "claim_generator": "http_resolver_cache", + "claim_generator_info": [ + {"name": "http_resolver_cache", "version": "0.1"}], + "format": "image/jpeg", + "title": "Signed with a caching HTTP resolver", + "assertions": [], + } + + # A fresh resolver per flow keeps the printed counts easy to follow. + resolver = CachingHttpResolver() + context = (c2pa.Context.builder() + .with_resolver(resolver) + .with_signer(c2pa.Signer.from_info(signer_info)) + .build()) + try: + builder = c2pa.Builder(manifest_definition, context=context) + + with open(fixtures_dir + "cloud.jpg", "rb") as ingredient: + for index in range(3): + ingredient.seek(0) + builder.add_ingredient( + {"title": f"cloud.jpg #{index + 1}"}, + "image/jpeg", ingredient) + + os.makedirs(output_dir, exist_ok=True) + output_path = output_dir + "A_signed_cached.jpg" + with open(fixtures_dir + "A.jpg", "rb") as source: + with open(output_path, "wb") as dest: + builder.sign( + c2pa.Signer.from_info(signer_info), + "image/jpeg", source, dest) + print(f"signed asset written to {output_path}") + finally: + context.close() + + print(f"cache hits={resolver.cache.hits} misses={resolver.cache.misses} " + "(expected 2 hits, 1 miss)") + + +def failing_resolver_is_a_clean_error(): + """A final failure surfaces as a typed error, not a crash.""" + print("\n--- A resolver that always answers 500 ---") + + def always_500(request): + return c2pa.HttpResponse(500, b"") + + with c2pa.Context.builder().with_resolver(always_500).build() as context: + try: + with open(fixtures_dir + "cloud.jpg", "rb") as f: + c2pa.Reader("image/jpeg", f, context=context) + print("unexpected success") + except c2pa.C2paError as e: + print(f"got a typed error as expected: {e}") + + +def main(): + try: + read_with_cache() + sign_with_cache() + except c2pa.C2paError as e: + message = str(e) + if "fetch" in message or "resolver" in message: + print(f"\nError: {e}") + print("This example needs internet access to fetch the remote " + "manifest for tests/fixtures/cloud.jpg.") + sys.exit(1) + raise + + # This one needs no network: the resolver answers without calling out. + failing_resolver_is_a_clean_error() + + +if __name__ == "__main__": + main() diff --git a/examples/http_resolver_debug.py b/examples/http_resolver_debug.py new file mode 100644 index 00000000..e7f6d775 --- /dev/null +++ b/examples/http_resolver_debug.py @@ -0,0 +1,158 @@ +import json +import os +import sys +import urllib.error +import urllib.request + +import c2pa + +# This example shows how to intercept every HTTP request the C2PA SDK makes, +# by attaching a custom HTTP resolver to a Context. The resolver logs the +# method and URL of each request and the status code of each response, while +# delegating the actual transfer to urllib from the standard library. +# +# A resolver sees all SDK HTTP traffic: remote manifest fetches, OCSP +# requests, RFC 3161 timestamp requests, and CAWG did:web resolution. +# +# This example needs internet access: tests/fixtures/cloud.jpg carries no +# embedded manifest, only a remote one that has to be fetched. + +fixtures_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "tests", "fixtures") + os.sep +output_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "output") + os.sep + + +class DebugHttpResolver: + """Logs every SDK HTTP request and response status. + + The actual transfer is delegated to urllib, which also gives us redirect + handling for free: the SDK itself does not follow redirects, so a + hand-rolled resolver would have to implement that. + """ + + def __init__(self, timeout=10.0): + self._timeout = timeout + + def resolve(self, request): + print(f"[http] {request.method} {request.url}", flush=True) + + # Timestamp requests POST a body; manifest fetches send none. + data = request.body or None + req = urllib.request.Request( + request.url, + data=data, + method=request.method, + headers=request.headers) + + try: + with urllib.request.urlopen(req, timeout=self._timeout) as resp: + body = resp.read() + print(f"[http] -> {resp.status} ({len(body)} bytes)", + flush=True) + return c2pa.HttpResponse(resp.status, body) + except urllib.error.HTTPError as e: + # A 4xx/5xx is still a real response. Pass the status through and + # let the SDK turn it into its own typed error: a remote manifest + # fetch only accepts 200. + body = e.read() + print(f"[http] -> {e.code}", flush=True) + return c2pa.HttpResponse(e.code, body) + # urllib.error.URLError (DNS failure, connection refused, timeout) is + # deliberately not caught: raising marks the request as a hard + # resolver failure, which surfaces as a typed C2paError. + + +def read_with_resolver(): + """Read an asset whose manifest lives at a remote URL.""" + print("\n--- Reading cloud.jpg (manifest is remote) ---") + + resolver = DebugHttpResolver() + with c2pa.Context.builder().with_resolver(resolver).build() as context: + with open(fixtures_dir + "cloud.jpg", "rb") as f: + with c2pa.Reader("image/jpeg", f, context=context) as reader: + print(f"validation state: {reader.get_validation_state()}") + print(f"embedded manifest: {reader.is_embedded()}") + print(f"remote URL: {reader.get_remote_url()}") + + +def sign_with_resolver(): + """Sign an asset, adding an ingredient whose manifest is remote.""" + print("\n--- Signing A.jpg with a remote-manifest ingredient ---") + + with open(fixtures_dir + "es256_certs.pem", "rb") as f: + certs = f.read() + with open(fixtures_dir + "es256_private.key", "rb") as f: + key = f.read() + + # ta_url is None, meaning "no timestamp authority". An empty string is + # treated as a URL and fails signing. + signer_info = c2pa.C2paSignerInfo( + alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) + + manifest_definition = { + "claim_generator": "http_resolver_debug", + "claim_generator_info": [ + {"name": "http_resolver_debug", "version": "0.1"}], + "format": "image/jpeg", + "title": "Signed with a debug HTTP resolver", + "assertions": [], + } + + resolver = DebugHttpResolver() + context = (c2pa.Context.builder() + .with_resolver(resolver) + .with_signer(c2pa.Signer.from_info(signer_info)) + .build()) + try: + builder = c2pa.Builder(manifest_definition, context=context) + + # Adding this ingredient fetches its remote manifest through the + # resolver, so one GET shows up in the log. + with open(fixtures_dir + "cloud.jpg", "rb") as ingredient: + builder.add_ingredient( + {"title": "cloud.jpg"}, "image/jpeg", ingredient) + + os.makedirs(output_dir, exist_ok=True) + output_path = output_dir + "A_signed_resolver.jpg" + with open(fixtures_dir + "A.jpg", "rb") as source: + with open(output_path, "wb") as dest: + builder.sign( + c2pa.Signer.from_info(signer_info), + "image/jpeg", source, dest) + print(f"signed asset written to {output_path}") + + # Reading the signed file back uses its embedded manifest, so this + # makes no HTTP requests at all. + print("\n--- Reading the signed asset (no HTTP expected) ---") + with open(output_path, "rb") as f: + with c2pa.Reader("image/jpeg", f) as reader: + store = json.loads(reader.json()) + manifest = store["manifests"][store["active_manifest"]] + for ingredient in manifest.get("ingredients", []): + print(f"ingredient: {ingredient.get('title')}") + finally: + context.close() + + +def main(): + try: + read_with_resolver() + sign_with_resolver() + except c2pa.C2paError as e: + message = str(e) + if "fetch" in message or "resolver" in message: + print(f"\nError: {e}") + print("This example needs internet access to fetch the remote " + "manifest for tests/fixtures/cloud.jpg.") + sys.exit(1) + raise + + print("\nNote: with a ta_url set on the signer, the log above would also " + "show the RFC 3161 timestamp POST.") + + +if __name__ == "__main__": + main() diff --git a/src/c2pa/__init__.py b/src/c2pa/__init__.py index bd581d58..9c7f6bff 100644 --- a/src/c2pa/__init__.py +++ b/src/c2pa/__init__.py @@ -31,6 +31,8 @@ Context, ContextBuilder, ContextProvider, + HttpRequest, + HttpResponse, sdk_version, load_settings ) # NOQA @@ -50,6 +52,8 @@ 'Context', 'ContextBuilder', 'ContextProvider', + 'HttpRequest', + 'HttpResponse', 'sdk_version', 'load_settings' ] diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 851348f8..e700f809 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -93,6 +93,10 @@ 'c2pa_context_builder_build', 'c2pa_context_builder_set_signer', 'c2pa_context_new', + # HTTP resolver bindings + 'c2pa_http_resolver_create', + 'c2pa_context_builder_set_http_resolver', + 'c2pa_error_set_last', # Free bindings 'c2pa_string_free', 'c2pa_free_string_array', @@ -646,6 +650,50 @@ def __del__(self): ctypes.c_ubyte), ctypes.c_size_t) +class C2paHttpRequest(ctypes.Structure): + """Mirror of the native C2paHttpRequest (#[repr(C)]). + + Read-only view: every pointer borrows native memory that is only valid + for the duration of the resolver callback. Copy anything you keep. + + The string fields are c_char_p so ctypes converts them to bytes on read. + `body` stays a c_ubyte pointer because it is binary, not NUL-terminated. + """ + _fields_ = [ + ("url", ctypes.c_char_p), + ("method", ctypes.c_char_p), + ("headers", ctypes.c_char_p), + ("body", ctypes.POINTER(ctypes.c_ubyte)), + ("body_len", ctypes.c_size_t), + ] + + +class C2paHttpResponse(ctypes.Structure): + """Mirror of the native C2paHttpResponse (#[repr(C)]). + + The callback fills this in. `body` must be allocated with the C runtime + malloc that the native library's free() matches: the native side takes + ownership and frees it on both the success and the error return path. + """ + _fields_ = [ + ("status", ctypes.c_int), + ("body", ctypes.POINTER(ctypes.c_ubyte)), + ("body_len", ctypes.c_size_t), + ] + + +class C2paHttpResolver(ctypes.Structure): + """Opaque structure for a native HTTP resolver handle.""" + _fields_ = [] # Empty as it's opaque in the C API + + +HttpResolverCallback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_void_p, + ctypes.POINTER(C2paHttpRequest), + ctypes.POINTER(C2paHttpResponse)) + + class StreamContext(ctypes.Structure): """Opaque structure for stream context.""" _fields_ = [] # Empty as it's opaque in the C API @@ -1041,6 +1089,23 @@ def _setup_function(func, argtypes, restype=None): [ctypes.POINTER(C2paContextBuilder), ctypes.POINTER(C2paSigner)], ctypes.c_int ) + +# HTTP resolver bindings +_setup_function( + _lib.c2pa_http_resolver_create, + [ctypes.c_void_p, HttpResolverCallback], + ctypes.POINTER(C2paHttpResolver) +) +_setup_function( + _lib.c2pa_context_builder_set_http_resolver, + [ctypes.POINTER(C2paContextBuilder), ctypes.POINTER(C2paHttpResolver)], + ctypes.c_int +) +_setup_function( + _lib.c2pa_error_set_last, + [ctypes.c_char_p], + ctypes.c_int +) _setup_function( _lib.c2pa_builder_sign_context, [ctypes.POINTER(C2paBuilder), @@ -1494,6 +1559,196 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: return mimetypes.guess_type(str(path))[0] or "" +_NATIVE_MALLOC = None + + +def _get_native_malloc(): + """Return malloc from the C runtime whose free() the native library calls. + + Resolver response bodies are handed to the native library, which frees + them with the platform libc free(). The allocation must come from the + matching runtime or the free is heap corruption, not a leak. + + Looked up lazily so importing c2pa never fails on an exotic platform + unless the resolver feature is actually used. + """ + global _NATIVE_MALLOC + if _NATIVE_MALLOC is None: + if sys.platform == "win32": + try: + # Rust MSVC targets link the UCRT, so libc::free is + # ucrtbase!free. The legacy msvcrt.dll is a different heap. + crt = ctypes.CDLL("ucrtbase") + except OSError: + crt = ctypes.CDLL("msvcrt") + else: + crt = ctypes.CDLL(None) + malloc = crt.malloc + malloc.argtypes = [ctypes.c_size_t] + # Required: the default c_int restype truncates 64-bit pointers. + malloc.restype = ctypes.c_void_p + _NATIVE_MALLOC = malloc + return _NATIVE_MALLOC + + +def _parse_header_lines(raw: str) -> dict: + """Parse the FFI's newline-delimited 'Name: Value' header block. + + The native side always sends a string (empty when there are no headers), + never NULL. Header names arrive lowercased, and repeated headers are sent + as separate lines, so the last occurrence of a name wins here. + """ + headers = {} + for line in raw.split("\n"): + name, sep, value = line.partition(":") + if sep: + headers[name.strip()] = value.strip() + return headers + + +class HttpRequest: + """An HTTP request the SDK asks a custom resolver to perform. + + Attributes: + url: Absolute request URL. + method: HTTP method ("GET", "POST", ...). + headers: Request headers as a dict. Names are lowercased by the + native layer; when a header repeats, the last value wins. + body: Request body bytes (b"" when there is none). Timestamp + requests POST a body; manifest fetches send none. + + All data is copied out of native memory, so it stays valid after the + resolver call returns. + """ + __slots__ = ("url", "method", "headers", "body") + + def __init__(self, url: str, method: str, headers: dict, body: bytes): + self.url = url + self.method = method + self.headers = headers + self.body = body + + def __repr__(self): + return (f"HttpRequest(method={self.method!r}, url={self.url!r}, " + f"body_len={len(self.body)})") + + +class HttpResponse: + """The answer a custom resolver returns to the SDK. + + Attributes: + status: HTTP status code. Remote manifest fetches only accept 200; + any other code surfaces as a typed C2paError. + body: Response body bytes. + """ + __slots__ = ("status", "body") + + def __init__(self, status: int, body: bytes = b""): + self.status = status + self.body = body + + def __repr__(self): + return (f"HttpResponse(status={self.status}, " + f"body_len={len(self.body or b'')})") + + +def _coerce_resolver(resolver): + """Normalize a resolver into a plain callable taking an HttpRequest. + + Accepts either an object with a resolve(request) method or a bare + callable with the same signature. + """ + resolve_fn = getattr(resolver, "resolve", None) + if callable(resolve_fn): + return resolve_fn + if callable(resolver): + return resolver + raise C2paError( + "HTTP resolver must be callable or provide a resolve() method") + + +def _make_http_resolver_trampoline(resolve_fn): + """Wrap a Python resolve function into a native C callback. + + Args: + resolve_fn: Callable[[HttpRequest], HttpResponse]. + + Returns: + The HttpResolverCallback object. The caller MUST keep a reference to + it for as long as any native context built from it can run: the + native side holds only a raw function pointer, and letting the thunk + be collected while a context can still call it is undefined behavior. + """ + def _trampoline(_ctx, request_ptr, response_ptr): + try: + req = request_ptr.contents + # Copy everything out now: these pointers borrow native memory + # that is only valid for the duration of this call. + url = req.url.decode("utf-8", "replace") if req.url else "" + method = (req.method.decode("utf-8", "replace") + if req.method else "") + raw_headers = (req.headers.decode("utf-8", "replace") + if req.headers else "") + body = (ctypes.string_at(req.body, req.body_len) + if (req.body and req.body_len) else b"") + + result = resolve_fn(HttpRequest( + url=url, + method=method, + headers=_parse_header_lines(raw_headers), + body=body)) + + payload = result.body or b"" + if not isinstance(payload, (bytes, bytearray)): + raise TypeError( + "HttpResponse.body must be bytes, got " + f"{type(payload).__name__}") + + response = response_ptr.contents + response.status = int(result.status) + if payload: + length = len(payload) + buf = _get_native_malloc()(length) + if not buf: + _lib.c2pa_error_set_last( + b"Other: HTTP resolver out of memory") + return -1 + ctypes.memmove(buf, bytes(payload), length) + # Ownership handoff: from here the native library frees this + # buffer on BOTH return paths (it copies then frees on 0, and + # frees a leftover body on non-zero). Never free it here. + response.body = ctypes.cast( + buf, ctypes.POINTER(ctypes.c_ubyte)) + response.body_len = length + else: + # body and body_len must stay NULL/0 together: the native + # side skips its free when body_len is 0, so a non-NULL + # pointer with a zero length is never freed and leaks. + response.body = None + response.body_len = 0 + return 0 + except BaseException as e: # noqa: B036 - must not unwind into native + # BaseException on purpose: an exception escaping a ctypes + # callback cannot propagate into Rust, so it would be reported + # as a generic failure with the real cause lost. Catching it + # here (including KeyboardInterrupt) turns it into a typed error + # carrying the actual message. + # + # Setting the error is mandatory, not best-effort: the native + # error slot is thread-local and is NOT cleared before the + # callback runs, so returning non-zero without setting it + # surfaces a stale, unrelated error from an earlier call. + try: + _lib.c2pa_error_set_last( + "Other: Python HTTP resolver failed: {}".format(e) + .encode("utf-8", "replace")) + except BaseException: + pass + return -1 + + return HttpResolverCallback(_trampoline) + + class ContextProvider(ABC): """Abstract base class for types that provide a C2PA context. @@ -1633,6 +1888,7 @@ class ContextBuilder: def __init__(self): self._settings = None self._signer = None + self._resolver = None def with_settings( self, settings: 'Settings', @@ -1660,11 +1916,59 @@ def with_signer( self._signer = signer return self + def with_resolver( + self, resolver, + ) -> 'ContextBuilder': + """Attach a custom HTTP resolver to the Context being built. + + The resolver handles every HTTP request the SDK makes through this + Context: remote manifest fetches, OCSP requests, RFC 3161 timestamp + requests, and CAWG did:web resolution. Reader and Builder instances + created without a Context keep using the built-in resolver. + + Can be called multiple times; the last resolver wins. + + Args: + resolver: An object with a resolve(request) -> HttpResponse + method, or a callable with the same signature. It receives + an HttpRequest and must return an HttpResponse. Raising + instead marks the request as a hard failure, which surfaces + as a typed C2paError. + + Returns: + self, for method chaining. + + Notes: + - Only status 200 is accepted for a remote manifest fetch; any + other status surfaces as a typed C2paError. + - The SDK does not follow redirects. A resolver delegating to + urllib.request gets redirect handling for free; a hand-rolled + one must implement it. + - Security: a custom resolver bypasses the + core.allowed_network_hosts setting, which only filters the + built-in resolver. Host filtering becomes the resolver's + responsibility. + - Settings still gate the resolver. With + verify.remote_manifest_fetch set to false the resolver is + never invoked and the read fails instead. + - Manifests larger than 10 MB are truncated without an error, + because the resolver response carries no Content-Length. + - The resolver may be called from SDK worker threads, so it must + be thread-safe, and it may be called for as long as any Reader + or Builder created from the Context is alive, including after + Context.close(). + - Do not call c2pa APIs from inside the resolver: reentering the + FFI while a call is in flight is undefined. + """ + self._resolver = resolver + return self + def build(self) -> 'Context': """Build and return a configured Context.""" return Context( settings=self._settings, signer=self._signer, + resolver=self._resolver, ) @@ -1693,10 +1997,24 @@ def __init__(self): _lib.c2pa_context_builder_new, "Failed to create ContextBuilder") + class _NativeHttpResolver(ManagedResource): + """Short-lived wrapper for a native C2paHttpResolver handle. + + Any failure inside its `with` block frees it via close(), unless a + consuming call already took ownership of it. + """ + + def __init__(self, callback_cb): + super().__init__() + self._create_and_activate( + lambda: _lib.c2pa_http_resolver_create(None, callback_cb), + "Failed to create HTTP resolver") + def __init__( self, settings: Optional['Settings'] = None, signer: Optional['Signer'] = None, + resolver=None, ): """Create a Context. @@ -1705,6 +2023,10 @@ def __init__( If None, default SDK settings are used. signer: Optional Signer. If provided it is consumed and must not be used directly again after that call. + resolver: Optional custom HTTP resolver, either an object with a + resolve(request) -> HttpResponse method or a callable with + the same signature. See ContextBuilder.with_resolver for the + full contract. Raises: C2paError: If creation fails @@ -1712,7 +2034,7 @@ def __init__( super().__init__() self._init_attrs() - if settings is None and signer is None: + if settings is None and signer is None and resolver is None: # Simple default context self._create_and_activate( _lib.c2pa_context_new, @@ -1728,6 +2050,19 @@ def __init__( "Failed to set settings on Context", check=lambda r: r != 0) + if resolver is not None: + resolve_fn = _coerce_resolver(resolver) + callback_cb = _make_http_resolver_trampoline(resolve_fn) + # Pin the thunk before any native call can capture its + # raw function pointer (see _release for why it stays). + self._http_resolver_cb = callback_cb + with self._NativeHttpResolver(callback_cb) as native_res: + native_res._consume_no_replacement( + lambda h: + _lib.c2pa_context_builder_set_http_resolver( + nb._handle, h), + "Failed to set HTTP resolver on Context: {}") + if signer is not None: signer._ensure_valid_state() # A rejected signer is retained, not closed and leaked. @@ -1748,9 +2083,26 @@ def _init_attrs(self): super()._init_attrs() self._has_signer = False self._signer_callback_cb = None + self._http_resolver_cb = None def _release(self): - """Release Context-specific resources.""" + """Release Context-specific resources. + + _http_resolver_cb is deliberately NOT cleared here. The native + context is an Arc, and c2pa_reader_from_context / + c2pa_builder_from_context clone it, so the native context (and the + resolver holding a raw pointer into the ctypes thunk) can outlive + this object's ACTIVE state. Reader and Builder keep a reference to + the Python Context for their whole lifetime, and that reference + chain is what keeps the thunk alive as long as any native clone can + still call it. Clearing it here frees the thunk while it is still + reachable from native code, which is undefined behavior: an observed + symptom is a garbage response reported as "invalid status code". + + The asymmetry with _signer_callback_cb below is intentional. That + one has the same latent hazard, but it predates this code and + changing it is out of scope here. + """ self._signer_callback_cb = None @classmethod @@ -1763,19 +2115,22 @@ def from_json( cls, json_str: str, signer: Optional['Signer'] = None, + resolver=None, ) -> 'Context': """Create Context from a JSON configuration string. Args: json_str: JSON string with settings config. signer: Optional Signer (consumed if provided). + resolver: Optional custom HTTP resolver. See + ContextBuilder.with_resolver for the contract. Returns: A new Context instance. """ settings = Settings.from_json(json_str) try: - return cls(settings=settings, signer=signer) + return cls(settings=settings, signer=signer, resolver=resolver) finally: settings.close() @@ -1784,17 +2139,21 @@ def from_dict( cls, config: dict, signer: Optional['Signer'] = None, + resolver=None, ) -> 'Context': """Create Context from a dictionary. Args: config: Dictionary with settings configuration. signer: Optional Signer (consumed if provided). + resolver: Optional custom HTTP resolver. See + ContextBuilder.with_resolver for the contract. Returns: A new Context instance. """ - return cls.from_json(json.dumps(config), signer=signer) + return cls.from_json( + json.dumps(config), signer=signer, resolver=resolver) @property def has_signer(self) -> bool: diff --git a/tests/test_http_resolver.py b/tests/test_http_resolver.py new file mode 100644 index 00000000..098b33a9 --- /dev/null +++ b/tests/test_http_resolver.py @@ -0,0 +1,553 @@ +# Copyright 2025 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, +# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) +# or the MIT license (http://opensource.org/licenses/MIT), +# at your option. + +# Unless required by applicable law or agreed to in writing, +# this software is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or +# implied. See the LICENSE-MIT and LICENSE-APACHE files for the +# specific language governing permissions and limitations under +# each license. + +"""Lifetime and memory tests for the custom HTTP resolver bindings. + +These tests are fully offline. The fixture is built once by signing a real +asset with set_no_embed() + set_remote_url(), which makes the SDK fetch the +manifest over HTTP; the test resolvers then serve those bytes from memory. + +Assertion discipline: never assert on exact native error text, because the +wording drifts between native releases. Assert only on the C2paError +(sub)type, the numeric status code, or a substring this test injected +itself. Likewise the signed fixture uses a test certificate that is not in +any trust list, so validation_state is Invalid by design: assert on the +presence of an active manifest, not on validity. +""" + +import ctypes +import gc +import io +import json +import os +import sys +import threading +import unittest + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + +from c2pa import ( # noqa: E402 + Builder, + C2paError, + C2paSignerInfo, + Context, + HttpResponse, + Reader, + Signer, +) +from c2pa.c2pa import ( # noqa: E402 + HttpResolverCallback, + _get_native_malloc, + _lib, + _parse_header_lines, +) + +FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") +REMOTE_URL = "http://manifests.example.test/m1.c2pa" + +MANIFEST_DEFINITION = { + "claim_generator": "python_test", + "claim_generator_info": [{"name": "python_test", "version": "0.1"}], + "format": "image/jpeg", + "title": "resolver test", + "assertions": [], +} + + +def _free_native_buffer(body_ptr): + """Free a buffer from _get_native_malloc that nothing else took over. + + Only for buffers the SDK never received: once a response body is handed + to the native layer, it owns it on every return path. + """ + crt = ctypes.CDLL("ucrtbase") if sys.platform == "win32" \ + else ctypes.CDLL(None) + crt.free.argtypes = [ctypes.c_void_p] + crt.free.restype = None + crt.free(ctypes.cast(body_ptr, ctypes.c_void_p)) + + +def _make_signer(): + """Build an es256 signer. + + ta_url is None, meaning "no timestamp authority": an empty string is + treated as a URL and fails signing with a Signature error. + """ + with open(os.path.join(FIXTURES, "es256_certs.pem"), "rb") as f: + certs = f.read() + with open(os.path.join(FIXTURES, "es256_private.key"), "rb") as f: + key = f.read() + return Signer.from_info(C2paSignerInfo(b"es256", certs, key, None)) + + +class CountingResolver: + """Resolver serving fixed bytes and recording every request.""" + + def __init__(self, manifest, status=200): + self._manifest = manifest + self._status = status + self._lock = threading.Lock() + self.requests = [] + + @property + def call_count(self): + with self._lock: + return len(self.requests) + + def resolve(self, request): + with self._lock: + self.requests.append(request) + return HttpResponse(self._status, self._manifest) + + +class HttpResolverTestBase(unittest.TestCase): + """Builds the offline remote-manifest fixture once for all tests.""" + + @classmethod + def setUpClass(cls): + builder = Builder(MANIFEST_DEFINITION) + builder.set_no_embed() + builder.set_remote_url(REMOTE_URL) + output = io.BytesIO() + with open(os.path.join(FIXTURES, "A.jpg"), "rb") as source: + cls.manifest_bytes = builder.sign( + _make_signer(), "image/jpeg", source, output) + cls.asset_bytes = output.getvalue() + + def asset_stream(self): + return io.BytesIO(self.asset_bytes) + + +class TestResolverInvocation(HttpResolverTestBase): + + def test_resolver_invoked_on_remote_read(self): + resolver = CountingResolver(self.manifest_bytes) + with Context.builder().with_resolver(resolver).build() as ctx: + reader = Reader("image/jpeg", self.asset_stream(), context=ctx) + manifest_store = json.loads(reader.json()) + + self.assertEqual(resolver.call_count, 1) + request = resolver.requests[0] + self.assertEqual(request.method, "GET") + self.assertEqual(request.url, REMOTE_URL) + # Request data is copied out of native memory, so it stays usable. + self.assertIsInstance(request.url, str) + self.assertIsInstance(request.method, str) + self.assertIsInstance(request.headers, dict) + self.assertIsInstance(request.body, bytes) + # A manifest fetch is a GET with no body. + self.assertEqual(request.body, b"") + self.assertTrue(manifest_store.get("active_manifest")) + + def test_with_resolver_accepts_callable_and_object(self): + calls = [] + + def resolve_fn(request): + calls.append(request) + return HttpResponse(200, self.manifest_bytes) + + with Context.builder().with_resolver(resolve_fn).build() as ctx: + Reader("image/jpeg", self.asset_stream(), context=ctx) + self.assertEqual(len(calls), 1) + + resolver = CountingResolver(self.manifest_bytes) + with Context.builder().with_resolver(resolver).build() as ctx: + Reader("image/jpeg", self.asset_stream(), context=ctx) + self.assertEqual(resolver.call_count, 1) + + def test_non_callable_resolver_rejected(self): + # Rejected while coercing, before any native resolver is created. + with self.assertRaises(C2paError): + Context.builder().with_resolver(42).build() + + def test_parse_header_lines(self): + # The native side sends an empty string, never NULL, when a request + # carries no headers. + self.assertEqual(_parse_header_lines(""), {}) + self.assertEqual( + _parse_header_lines("accept: */*\nx-token: abc\n"), + {"accept": "*/*", "x-token": "abc"}) + # Repeated names arrive on separate lines; the last one wins. + self.assertEqual( + _parse_header_lines("a: 1\na: 2\n"), {"a": "2"}) + + +class TestResolverErrorPaths(HttpResolverTestBase): + + def test_non_200_statuses(self): + for status in (404, 500, 204, 301): + with self.subTest(status=status): + body = b"boom" if status == 500 else b"" + + def resolve_fn(request, status=status, body=body): + return HttpResponse(status, body) + + with Context.builder().with_resolver( + resolve_fn).build() as ctx: + with self.assertRaises(C2paError) as caught: + Reader("image/jpeg", self.asset_stream(), context=ctx) + # The status code is a stable signal; the wording is not. + self.assertIn(str(status), str(caught.exception)) + + # The process is still healthy afterwards. + resolver = CountingResolver(self.manifest_bytes) + with Context.builder().with_resolver(resolver).build() as ctx: + Reader("image/jpeg", self.asset_stream(), context=ctx) + self.assertEqual(resolver.call_count, 1) + + def test_resolver_exception_is_contained(self): + sentinel = "resolver_sentinel_9f3a" + + def resolve_fn(request): + raise RuntimeError(sentinel) + + with Context.builder().with_resolver(resolve_fn).build() as ctx: + with self.assertRaises(C2paError) as caught: + Reader("image/jpeg", self.asset_stream(), context=ctx) + # Only assert on text this test injected itself. + self.assertIn(sentinel, str(caught.exception)) + + # The error slot is not sticky: a fresh context still works. + resolver = CountingResolver(self.manifest_bytes) + with Context.builder().with_resolver(resolver).build() as ctx: + reader = Reader("image/jpeg", self.asset_stream(), context=ctx) + self.assertTrue(json.loads(reader.json()).get("active_manifest")) + + def test_resolver_bad_return_types(self): + cases = { + "none": lambda request: None, + "no_status_attr": lambda request: object(), + "str_body": lambda request: HttpResponse(200, "not-bytes"), + } + for name, resolve_fn in cases.items(): + with self.subTest(case=name): + with Context.builder().with_resolver( + resolve_fn).build() as ctx: + # Contained as a typed error, never a crash. + with self.assertRaises(C2paError): + Reader("image/jpeg", self.asset_stream(), context=ctx) + + +class TestResolverLifetime(HttpResolverTestBase): + + def test_callback_alive_after_context_close(self): + """The resolver must survive Context.close(). + + The native context is an Arc that Builder clones, so the resolver + can be called after the Python Context is closed. Regression test + for keeping _http_resolver_cb alive in _release(). + """ + resolver = CountingResolver(self.manifest_bytes) + ctx = Context.builder().with_resolver(resolver).build() + builder = Builder(MANIFEST_DEFINITION, context=ctx) + + ctx.close() + del ctx + gc.collect() + + builder.add_ingredient( + {"title": "remote ingredient"}, "image/jpeg", self.asset_stream()) + self.assertEqual(resolver.call_count, 1) + + def test_context_lifecycle_with_resolver(self): + resolver = CountingResolver(self.manifest_bytes) + ctx = Context.builder().with_resolver(resolver).build() + self.assertTrue(ctx.is_valid) + + ctx.close() + ctx.close() # idempotent + self.assertFalse(ctx.is_valid) + with self.assertRaises(C2paError): + ctx._ensure_valid_state() + + with Context.builder().with_resolver(resolver).build() as ctx2: + self.assertTrue(ctx2.is_valid) + + def test_init_attrs_invariant(self): + # _init_attrs must define the attribute, so instances built by + # _wrap_native_handle (which skips __init__) always have it. + ctx = Context() + try: + self.assertIsNone(ctx._http_resolver_cb) + finally: + ctx.close() + + def test_shared_context_threaded(self): + resolver = CountingResolver(self.manifest_bytes) + errors = [] + + def read_once(): + try: + Reader("image/jpeg", self.asset_stream(), context=ctx) + except BaseException as e: # noqa: B036 - report, don't swallow + errors.append(e) + + with Context.builder().with_resolver(resolver).build() as ctx: + threads = [threading.Thread(target=read_once) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + self.assertEqual(errors, []) + self.assertGreaterEqual(resolver.call_count, 8) + + def test_default_paths_untouched(self): + # No context at all: the built-in resolver path is unchanged. + with open(os.path.join(FIXTURES, "C.jpg"), "rb") as f: + reader = Reader("image/jpeg", f) + self.assertTrue(json.loads(reader.json()).get("active_manifest")) + + # A context with no resolver still takes the c2pa_context_new fast + # path. + with Context() as ctx: + self.assertTrue(ctx.is_valid) + + +class TestResolverSettingsInteraction(HttpResolverTestBase): + + def test_settings_gate_resolver(self): + resolver = CountingResolver(self.manifest_bytes) + ctx = Context.from_dict( + {"verify": {"remote_manifest_fetch": False}}, resolver=resolver) + try: + with self.assertRaises(C2paError): + Reader("image/jpeg", self.asset_stream(), context=ctx) + # Settings gate the resolver: it is never consulted. + self.assertEqual(resolver.call_count, 0) + finally: + ctx.close() + + def test_settings_resolver_and_signer_together(self): + """The full builder path: settings, then resolver, then signer.""" + resolver = CountingResolver(self.manifest_bytes) + ctx = Context.from_dict( + {"verify": {"remote_manifest_fetch": True}}, + signer=_make_signer(), + resolver=resolver) + try: + builder = Builder(MANIFEST_DEFINITION, context=ctx) + builder.add_ingredient( + {"title": "first"}, "image/jpeg", self.asset_stream()) + builder.add_ingredient( + {"title": "second"}, "image/jpeg", self.asset_stream()) + self.assertEqual(resolver.call_count, 2) + + output = io.BytesIO() + with open(os.path.join(FIXTURES, "A.jpg"), "rb") as source: + builder.sign(_make_signer(), "image/jpeg", source, output) + self.assertGreater(len(output.getvalue()), 0) + + # Reading the signed output back uses the embedded manifest, so + # it needs no further fetches. + before = resolver.call_count + reader = Reader( + "image/jpeg", io.BytesIO(output.getvalue()), context=ctx) + self.assertTrue(json.loads(reader.json()).get("active_manifest")) + self.assertEqual(resolver.call_count, before) + finally: + ctx.close() + + +def _current_rss_mb(): + """Resident set size of this process, in MB. + + Deliberately not resource.getrusage(): ru_maxrss is a high-water mark, + so once a peak is reached a later leak stays hidden underneath it. A + leak sentinel needs the current value. + """ + with open("/proc/self/statm") as f: + return int(f.read().split()[1]) * os.sysconf("SC_PAGE_SIZE") / 1048576 + + +def _current_rss_mb_darwin(): + import subprocess + out = subprocess.run( + ["ps", "-o", "rss=", "-p", str(os.getpid())], + capture_output=True, text=True, check=True) + return int(out.stdout.strip()) / 1024 + + +@unittest.skipUnless(sys.platform in ("linux", "darwin"), + "RSS sampling is implemented for Linux and macOS only") +class TestResolverMemory(HttpResolverTestBase): + """Leak sentinels for the response-body ownership contract. + + The native library frees the response body on BOTH return paths, so the + binding must hand the buffer over and never free it afterwards. True + growth is ~0 MB; the threshold only absorbs allocator noise. + + The payload is written into every buffer on purpose. malloc alone does + not commit pages, so an untouched leak would not move RSS and the + sentinel would pass while leaking. Calibrated by injecting the + body_len == 0 leak: 200 iterations of 256 KB grows ~50 MB, well clear + of the threshold. + """ + + ITERATIONS = 200 + CHUNK = 256 * 1024 + THRESHOLD_MB = 20 + + @staticmethod + def _rss_mb(): + if sys.platform == "darwin": + return _current_rss_mb_darwin() + return _current_rss_mb() + + def _assert_rss_stable(self, run_once, label): + # Warm up so first-call allocations are not counted as growth. + for _ in range(5): + run_once() + gc.collect() + before = self._rss_mb() + for _ in range(self.ITERATIONS): + run_once() + gc.collect() + growth = self._rss_mb() - before + would_leak = self.ITERATIONS * self.CHUNK / 1048576 + self.assertLess( + growth, self.THRESHOLD_MB, + f"{label}: RSS grew {growth:.1f} MB " + f"(a full leak would be ~{would_leak:.0f} MB)") + + def test_repeated_cycles_no_leak(self): + """Success path: the native side copies the body then frees it.""" + payload = b"x" * self.CHUNK + + def resolve_fn(request): + # Not a valid manifest, so the read fails; the body is still + # handed to the native side and must still be freed. + return HttpResponse(200, payload) + + def run_once(): + with Context.builder().with_resolver(resolve_fn).build() as ctx: + try: + Reader("image/jpeg", self.asset_stream(), context=ctx) + except C2paError: + pass + + self._assert_rss_stable(run_once, "success path") + + def test_valid_manifest_cycles_no_leak(self): + """Success path with a real manifest, which is fully parsed.""" + resolver = CountingResolver(self.manifest_bytes) + + def run_once(): + with Context.builder().with_resolver(resolver).build() as ctx: + Reader("image/jpeg", self.asset_stream(), context=ctx) + + self._assert_rss_stable(run_once, "valid manifest") + + def test_error_path_body_is_freed(self): + """Error path: a body left behind on a non-zero return is freed. + + Uses a raw callback so the buffer is deliberately assigned before + returning -1, which the binding's own trampoline never does. + """ + chunk = self.CHUNK + malloc = _get_native_malloc() + + def raw_callback(_ctx, request_ptr, response_ptr): + buf = malloc(chunk) + ctypes.memmove(buf, b"x" * chunk, chunk) + response = response_ptr.contents + response.body = ctypes.cast(buf, ctypes.POINTER(ctypes.c_ubyte)) + response.body_len = chunk + _lib.c2pa_error_set_last(b"Other: forced resolver failure") + return -1 + + callback_cb = HttpResolverCallback(raw_callback) + resolver_ptr = _lib.c2pa_http_resolver_create(None, callback_cb) + self.assertTrue(resolver_ptr) + + builder_ptr = _lib.c2pa_context_builder_new() + self.assertEqual( + _lib.c2pa_context_builder_set_http_resolver( + builder_ptr, resolver_ptr), 0) + context_ptr = _lib.c2pa_context_builder_build(builder_ptr) + self.assertTrue(context_ptr) + + wrapped = Context._wrap_native_handle(context_ptr) + # Keep the thunk alive for as long as the context can call it. + wrapped._http_resolver_cb = callback_cb + try: + def run_once(): + try: + Reader("image/jpeg", self.asset_stream(), context=wrapped) + except C2paError: + pass + + self._assert_rss_stable(run_once, "error path") + finally: + wrapped.close() + + def test_empty_body_is_null_not_zero_length(self): + """An empty body must be emitted as NULL, not a zero-length buffer. + + The native side skips its free when body_len == 0, so a non-NULL + pointer with a zero length is never freed and leaks. Rather than + infer this from RSS, drive the trampoline directly and inspect the + response struct it fills in. + """ + from c2pa.c2pa import (C2paHttpRequest, C2paHttpResponse, + _make_http_resolver_trampoline) + + callback_cb = _make_http_resolver_trampoline( + lambda request: HttpResponse(200, b"")) + + request = C2paHttpRequest( + url=b"http://example.test/m.c2pa", method=b"GET", + headers=b"", body=None, body_len=0) + response = C2paHttpResponse(status=0, body=None, body_len=0) + + rc = callback_cb( + None, ctypes.byref(request), ctypes.byref(response)) + + self.assertEqual(rc, 0) + self.assertEqual(response.status, 200) + self.assertEqual(response.body_len, 0) + # The pointer must be NULL, so the native layer has nothing to free. + self.assertFalse(bool(response.body)) + + def test_non_empty_body_is_handed_over(self): + """A non-empty body arrives as a native buffer of the right size.""" + from c2pa.c2pa import (C2paHttpRequest, C2paHttpResponse, + _make_http_resolver_trampoline) + + payload = b"manifest-bytes" + callback_cb = _make_http_resolver_trampoline( + lambda request: HttpResponse(200, payload)) + + request = C2paHttpRequest( + url=b"http://example.test/m.c2pa", method=b"GET", + headers=b"", body=None, body_len=0) + response = C2paHttpResponse(status=0, body=None, body_len=0) + + rc = callback_cb( + None, ctypes.byref(request), ctypes.byref(response)) + try: + self.assertEqual(rc, 0) + self.assertEqual(response.body_len, len(payload)) + self.assertTrue(bool(response.body)) + self.assertEqual( + ctypes.string_at(response.body, response.body_len), payload) + finally: + # Nothing consumed this buffer, so free it here to keep the test + # itself leak-free. Never do this once the SDK has taken it. + if response.body: + _free_native_buffer(response.body) + + +if __name__ == "__main__": + unittest.main() From 6d3e0c7ede81584459f690f3db88c4182707490d Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 11:04:23 -0700 Subject: [PATCH 40/67] fix: formatting --- docs/native-resources-management.md | 4 ++-- examples/README.md | 16 ++++++++-------- examples/http_resolver_debug.py | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 52a10981..4694fb90 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -495,7 +495,7 @@ The cleanup order matters: `_release()` runs first (closing streams, dropping ca ### Dropping a Context reference -`Reader` and `Builder` both keep a `_context` attribute that is written once and never read. It is not dead code: it is what keeps the Context alive while the native handle depends on it. Without that reference, `Reader("image/jpeg", stream, context=Context())` would let the Context become collectable as soon as the constructor returned, even though the reader is still using it. +`Reader` and `Builder` both keep a `_context` attribute that is written once and never read. The reference keeps the Context alive while the native handle depends on it. Without it, `Reader("image/jpeg", stream, context=Context())` would let the Context become collectable as soon as the constructor returned, even though the reader is still using it. Clearing it in `_release()` is the other half of that. A closed Reader has no further use for the Context, and holding the reference would keep alive an object nothing can reach through the Reader's public API. @@ -530,7 +530,7 @@ sequenceDiagram Both `_cleanup_resources()` and the consumed teardown take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. -The memory the child skips is not lost for good. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. +The memory the child skips is reclaimed anyway. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. > [!NOTE] > `is_foreign_process()` returns `False` when no owner PID was ever recorded, so an object that somehow missed the stamp is cleaned up as before rather than leaking silently. diff --git a/examples/README.md b/examples/README.md index e9ef242a..faf59f70 100644 --- a/examples/README.md +++ b/examples/README.md @@ -70,12 +70,12 @@ except Exception as err: ## Using a custom HTTP resolver -A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`: remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG `did:web` resolution. Use it to add authentication headers, cache responses, log traffic, or serve responses from memory in tests. +A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. A resolver is either an object with a `resolve(request)` method or a plain callable. It receives an `HttpRequest` and returns an `HttpResponse`: ```py -class MyResolver: +class AnHttpResolver: def resolve(self, request): # request.url, request.method, request.headers, request.body return c2pa.HttpResponse(200, b"...") @@ -84,18 +84,18 @@ context = c2pa.Context.builder().with_resolver(MyResolver()).build() reader = c2pa.Reader("image/jpeg", stream, context=context) ``` -Raising from the resolver marks the request as a hard failure; returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. +Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. -Two things are worth knowing before you write one: +Two things to note before writing one: - A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. -- The SDK does not follow redirects. Delegating to `urllib.request` gives you redirect handling for free. +- The SDK does not follow redirects by default. Delegating to `urllib.request` in the examples gives you redirect handling for free. -The [`examples/http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_debug.py) script logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. It runs one read flow and one signing flow. +The [`examples/http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_debug.py) script logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. -The [`examples/http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_cache.py) script adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. It reads the same asset twice to show a cache hit, then adds the same remote-manifest ingredient three times while signing, which produces one network fetch and two cache hits. +The [`examples/http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_cache.py) script adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. -Both scripts use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so **they need internet access**. If the fetch fails they print a hint instead of a traceback. +Both scripts use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access. ## Running the examples diff --git a/examples/http_resolver_debug.py b/examples/http_resolver_debug.py index e7f6d775..a174214f 100644 --- a/examples/http_resolver_debug.py +++ b/examples/http_resolver_debug.py @@ -67,7 +67,7 @@ def resolve(self, request): def read_with_resolver(): """Read an asset whose manifest lives at a remote URL.""" - print("\n--- Reading cloud.jpg (manifest is remote) ---") + print("\n--- Reading cloud.jpg (manifest is remote):") resolver = DebugHttpResolver() with c2pa.Context.builder().with_resolver(resolver).build() as context: @@ -80,7 +80,7 @@ def read_with_resolver(): def sign_with_resolver(): """Sign an asset, adding an ingredient whose manifest is remote.""" - print("\n--- Signing A.jpg with a remote-manifest ingredient ---") + print("\n--- Signing A.jpg with a remote-manifest ingredient:") with open(fixtures_dir + "es256_certs.pem", "rb") as f: certs = f.read() @@ -124,9 +124,9 @@ def sign_with_resolver(): "image/jpeg", source, dest) print(f"signed asset written to {output_path}") - # Reading the signed file back uses its embedded manifest, so this - # makes no HTTP requests at all. - print("\n--- Reading the signed asset (no HTTP expected) ---") + # Reading the signed file back uses its embedded manifest, + # so this makes no HTTP requests at all. + print("\n--- Reading the signed asset (no HTTP expected): ") with open(output_path, "rb") as f: with c2pa.Reader("image/jpeg", f) as reader: store = json.loads(reader.json()) From a8f56d740f0ad5f33dcb3adb031544ff80dd0229 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 11:38:48 -0700 Subject: [PATCH 41/67] fix: Examples --- examples/http_resolver_cache.py | 63 ++++++++++++++++++++------------- examples/http_resolver_debug.py | 56 ++++++++++++++++------------- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/examples/http_resolver_cache.py b/examples/http_resolver_cache.py index e354d828..739c0df7 100644 --- a/examples/http_resolver_cache.py +++ b/examples/http_resolver_cache.py @@ -8,14 +8,8 @@ import c2pa -# This example shows a custom HTTP resolver that caches responses and retries -# throttled requests, using nothing but the standard library. -# -# The SDK may call a resolver from worker threads, so everything here is -# thread-safe. -# -# This example needs internet access: tests/fixtures/cloud.jpg carries no -# embedded manifest, only a remote one that has to be fetched. +# This example shows a custom HTTP resolver that caches responses +# and retries throttled requests. fixtures_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), @@ -61,13 +55,14 @@ def put(self, key, value): class CachingHttpResolver: """An HTTP resolver with a response cache and bounded retries. - Caching policy: only GET requests answered with 200 are cached. POSTs - (timestamp requests) and error responses never are. + Caching policy: only read GET requests answered with 200 are cached. + POSTs (timestamp requests) and error responses are not. - Retry policy: 429 and 503 are retried up to max_retries times, honoring - a capped Retry-After when the header is present and otherwise backing - off exponentially. Any other status is final and is passed through to - the SDK. Transport errors raise, which marks a hard failure. + Retry policy: 429 and 503 are retried up to max_retries times, + honoring a capped Retry-After when the header is present, + and otherwise backing off exponentially. + Any other status is final and is passed through to the SDK. + Transport errors raise, which marks a hard failure. """ def __init__(self, cache=None, timeout=10.0, max_retries=3, @@ -107,8 +102,8 @@ def _fetch_with_retries(self, request): except urllib.error.HTTPError as e: retryable = e.code in (429, 503) if not retryable or attempt == self._max_retries: - # Final failure: pass the status back so the SDK can - # report it as a typed error. Not cached. + # Final failure: + # Pass the status back so the SDK can report it.. return e.code, e.read() delay = self._retry_delay(e, attempt) print(f"[http] {e.code}, retrying in {delay:.1f}s", @@ -127,8 +122,8 @@ def _retry_delay(self, error, attempt): def read_with_cache(): - """Read the same remote-manifest asset twice: one fetch, one cache hit.""" - print("\n--- Reading cloud.jpg twice through one cache ---") + """Read the same remote-manifest asset more than once for cache hits.""" + print("\n--- Reading cloud.jpg multiple times should hit a cache") resolver = CachingHttpResolver() with c2pa.Context.builder().with_resolver(resolver).build() as context: @@ -147,28 +142,44 @@ def sign_with_cache(): Each add re-reads the ingredient's remote manifest through the context resolver, so three adds mean one network fetch and two cache hits. """ - print("\n--- Signing with the same remote ingredient added 3 times ---") + print("\n--- Signing with the same remote ingredient added multiple times...") with open(fixtures_dir + "es256_certs.pem", "rb") as f: certs = f.read() with open(fixtures_dir + "es256_private.key", "rb") as f: key = f.read() - # ta_url is None, meaning "no timestamp authority". An empty string is - # treated as a URL and fails signing. + # ta_url is None, meaning "no timestamp authority". + # An empty string is treated as a URL and fails signing. signer_info = c2pa.C2paSignerInfo( alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) + ingredient_labels = [f"cloud-ingredient-{i + 1}" for i in range(3)] + manifest_definition = { "claim_generator": "http_resolver_cache", "claim_generator_info": [ {"name": "http_resolver_cache", "version": "0.1"}], "format": "image/jpeg", "title": "Signed with a caching HTTP resolver", - "assertions": [], + "assertions": [{ + "label": "c2pa.actions.v2", + "data": {"actions": [ + { + "action": "c2pa.created", + "digitalSourceType": + "http://cv.iptc.org/newscodes/" + "digitalsourcetype/digitalCreation", + }, + { + "action": "c2pa.placed", + "parameters": {"ingredientIds": ingredient_labels}, + }, + ]}, + }], } - # A fresh resolver per flow keeps the printed counts easy to follow. + # A fresh resolver per flow so it prints nicely. resolver = CachingHttpResolver() context = (c2pa.Context.builder() .with_resolver(resolver) @@ -181,7 +192,9 @@ def sign_with_cache(): for index in range(3): ingredient.seek(0) builder.add_ingredient( - {"title": f"cloud.jpg #{index + 1}"}, + {"title": f"cloud.jpg #{index + 1}", + "relationship": "componentOf", + "label": ingredient_labels[index]}, "image/jpeg", ingredient) os.makedirs(output_dir, exist_ok=True) @@ -201,7 +214,7 @@ def sign_with_cache(): def failing_resolver_is_a_clean_error(): """A final failure surfaces as a typed error, not a crash.""" - print("\n--- A resolver that always answers 500 ---") + print("\n--- A resolver that always answers 500") def always_500(request): return c2pa.HttpResponse(500, b"") diff --git a/examples/http_resolver_debug.py b/examples/http_resolver_debug.py index a174214f..3e726d09 100644 --- a/examples/http_resolver_debug.py +++ b/examples/http_resolver_debug.py @@ -7,15 +7,10 @@ import c2pa # This example shows how to intercept every HTTP request the C2PA SDK makes, -# by attaching a custom HTTP resolver to a Context. The resolver logs the -# method and URL of each request and the status code of each response, while -# delegating the actual transfer to urllib from the standard library. -# -# A resolver sees all SDK HTTP traffic: remote manifest fetches, OCSP -# requests, RFC 3161 timestamp requests, and CAWG did:web resolution. -# -# This example needs internet access: tests/fixtures/cloud.jpg carries no -# embedded manifest, only a remote one that has to be fetched. +# by attaching a custom HTTP resolver to a Context. +# The resolver logs the method and URL of each request and the status code +# of each response, while delegating the actual transfer to urllib from +# the standard library. The resolver sees all network traffic. fixtures_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), @@ -27,10 +22,7 @@ class DebugHttpResolver: """Logs every SDK HTTP request and response status. - - The actual transfer is delegated to urllib, which also gives us redirect - handling for free: the SDK itself does not follow redirects, so a - hand-rolled resolver would have to implement that. + The actual transfer is delegated to urllib. """ def __init__(self, timeout=10.0): @@ -54,15 +46,14 @@ def resolve(self, request): flush=True) return c2pa.HttpResponse(resp.status, body) except urllib.error.HTTPError as e: - # A 4xx/5xx is still a real response. Pass the status through and - # let the SDK turn it into its own typed error: a remote manifest - # fetch only accepts 200. + # A 4xx/5xx is still a response! + # Pass the status through and let the SDK turn it into + # its own typed error: a remote manifest fetch only accepts 200. body = e.read() print(f"[http] -> {e.code}", flush=True) return c2pa.HttpResponse(e.code, body) - # urllib.error.URLError (DNS failure, connection refused, timeout) is - # deliberately not caught: raising marks the request as a hard - # resolver failure, which surfaces as a typed C2paError. + # urllib.error.URLError (DNS failure, connection refused, timeout) is deliberately not caught: + # raising marks the request as a hard resolver failure, which surfaces as a typed C2paError. def read_with_resolver(): @@ -87,8 +78,7 @@ def sign_with_resolver(): with open(fixtures_dir + "es256_private.key", "rb") as f: key = f.read() - # ta_url is None, meaning "no timestamp authority". An empty string is - # treated as a URL and fails signing. + # ta_url is None, meaning "no timestamp authority". signer_info = c2pa.C2paSignerInfo( alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) @@ -98,7 +88,21 @@ def sign_with_resolver(): {"name": "http_resolver_debug", "version": "0.1"}], "format": "image/jpeg", "title": "Signed with a debug HTTP resolver", - "assertions": [], + "assertions": [{ + "label": "c2pa.actions.v2", + "data": {"actions": [ + { + "action": "c2pa.created", + "digitalSourceType": + "http://cv.iptc.org/newscodes/" + "digitalsourcetype/digitalCreation", + }, + { + "action": "c2pa.placed", + "parameters": {"ingredientIds": ["cloud-ingredient"]}, + }, + ]}, + }], } resolver = DebugHttpResolver() @@ -109,11 +113,13 @@ def sign_with_resolver(): try: builder = c2pa.Builder(manifest_definition, context=context) - # Adding this ingredient fetches its remote manifest through the - # resolver, so one GET shows up in the log. + # Adding this ingredient fetches its remote manifest through the resolver, + # so one GET shows up in the log. with open(fixtures_dir + "cloud.jpg", "rb") as ingredient: builder.add_ingredient( - {"title": "cloud.jpg"}, "image/jpeg", ingredient) + {"title": "cloud.jpg", "relationship": "componentOf", + "label": "cloud-ingredient"}, + "image/jpeg", ingredient) os.makedirs(output_dir, exist_ok=True) output_path = output_dir + "A_signed_resolver.jpg" From 165db90b8badc91df7415ba96a7b8ac4d9f7a7a0 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 13:11:57 -0700 Subject: [PATCH 42/67] fix: Refactor --- examples/README.md | 38 +- examples/http_resolver_cache.py | 249 ---------- examples/http_resolver_debug.py | 164 ------- src/c2pa/__init__.py | 4 - src/c2pa/c2pa.py | 514 ++++++++++---------- tests/network/__init__.py | 1 + tests/network/test_http_resolver_cache.py | 283 +++++++++++ tests/network/test_http_resolver_debug.py | 206 ++++++++ tests/test_http_resolver.py | 553 ---------------------- 9 files changed, 773 insertions(+), 1239 deletions(-) delete mode 100644 examples/http_resolver_cache.py delete mode 100644 examples/http_resolver_debug.py create mode 100644 tests/network/__init__.py create mode 100644 tests/network/test_http_resolver_cache.py create mode 100644 tests/network/test_http_resolver_debug.py delete mode 100644 tests/test_http_resolver.py diff --git a/examples/README.md b/examples/README.md index faf59f70..cd8d2f8f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -72,13 +72,13 @@ except Exception as err: A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. -A resolver is either an object with a `resolve(request)` method or a plain callable. It receives an `HttpRequest` and returns an `HttpResponse`: +A resolver is either an object with a `resolve(request)` method or a plain callable. It receives a request object exposing `.url`, `.method`, `.headers` (dict), and `.body` (bytes), and must return a response object exposing `.status` (int) and `.body` (bytes) — any object with those attributes works, there is no type to import: ```py class AnHttpResolver: def resolve(self, request): # request.url, request.method, request.headers, request.body - return c2pa.HttpResponse(200, b"...") + return SomeResponse(status=200, body=b"...") context = c2pa.Context.builder().with_resolver(MyResolver()).build() reader = c2pa.Reader("image/jpeg", stream, context=context) @@ -89,13 +89,19 @@ Raising from the resolver marks the request as a hard failure. Returning a non-2 Two things to note before writing one: - A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. -- The SDK does not follow redirects by default. Delegating to `urllib.request` in the examples gives you redirect handling for free. +- The SDK does not follow redirects by default. Delegating to `urllib.request` in the examples below gives you redirect handling for free. -The [`examples/http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_debug.py) script logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. +Custom resolvers need a live remote manifest to demonstrate against, so their examples live as runnable tests rather than standalone scripts: -The [`examples/http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/examples/http_resolver_cache.py) script adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. +The [`tests/network/test_http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_debug.py) test logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. -Both scripts use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access. +The [`tests/network/test_http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_cache.py) test adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. + +Both use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when it can't reach the network. Run them with: + +```bash +python -m pytest tests/network/ -v +``` ## Running the examples @@ -129,25 +135,7 @@ In this example, `SignerInfo` creates a `Signer` object that signs the manifest. python examples/sign_info.py ``` -### Run the debugging HTTP resolver example - -This example logs every HTTP request the SDK makes. It needs internet access. - -```bash -python examples/http_resolver_debug.py -``` - -Expected output: one `GET` of the Adobe manifest URL with a `-> 200` for the read flow, another `GET` when the remote-manifest ingredient is added while signing, and no HTTP at all when the signed result is read back (its manifest is embedded). - -### Run the caching HTTP resolver example - -This example caches responses and retries throttled requests. It needs internet access. - -```bash -python examples/http_resolver_cache.py -``` - -Expected output: `MISS` then `HIT` for the two reads, then `1 miss` and `2 hits` for the three ingredient additions during signing. It ends with a resolver that always answers 500, showing that a final failure surfaces as a typed `C2paError` rather than a crash. +See [Using a custom HTTP resolver](#using-a-custom-http-resolver) above for the debugging and caching resolver examples — they need internet access, so they live under `tests/network/` as runnable (skip-if-offline) tests instead of standalone scripts here. ## Backend application example diff --git a/examples/http_resolver_cache.py b/examples/http_resolver_cache.py deleted file mode 100644 index 739c0df7..00000000 --- a/examples/http_resolver_cache.py +++ /dev/null @@ -1,249 +0,0 @@ -import collections -import os -import sys -import threading -import time -import urllib.error -import urllib.request - -import c2pa - -# This example shows a custom HTTP resolver that caches responses -# and retries throttled requests. - -fixtures_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "tests", "fixtures") + os.sep -output_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "output") + os.sep - - -class TtlLruCache: - """A small LRU cache whose entries also expire after a TTL.""" - - def __init__(self, max_items=100, ttl_seconds=120.0): - self._max_items = int(max_items) - self._ttl = float(ttl_seconds) - self._entries = collections.OrderedDict() # key -> (expiry, value) - self._lock = threading.Lock() - self.hits = 0 - self.misses = 0 - - def get(self, key): - with self._lock: - entry = self._entries.get(key) - # time.monotonic, not time.time: immune to wall-clock jumps. - if entry is not None and entry[0] > time.monotonic(): - self._entries.move_to_end(key) - self.hits += 1 - return entry[1] - if entry is not None: - del self._entries[key] - self.misses += 1 - return None - - def put(self, key, value): - with self._lock: - if key in self._entries: - self._entries.move_to_end(key) - self._entries[key] = (time.monotonic() + self._ttl, value) - while len(self._entries) > self._max_items: - self._entries.popitem(last=False) - - -class CachingHttpResolver: - """An HTTP resolver with a response cache and bounded retries. - - Caching policy: only read GET requests answered with 200 are cached. - POSTs (timestamp requests) and error responses are not. - - Retry policy: 429 and 503 are retried up to max_retries times, - honoring a capped Retry-After when the header is present, - and otherwise backing off exponentially. - Any other status is final and is passed through to the SDK. - Transport errors raise, which marks a hard failure. - """ - - def __init__(self, cache=None, timeout=10.0, max_retries=3, - backoff_seconds=0.5, max_retry_after=10.0): - self.cache = cache if cache is not None else TtlLruCache() - self._timeout = timeout - self._max_retries = int(max_retries) - self._backoff = float(backoff_seconds) - self._max_retry_after = float(max_retry_after) - - def resolve(self, request): - cacheable = request.method.upper() == "GET" - if cacheable: - cached = self.cache.get(request.url) - if cached is not None: - print(f"[cache] HIT {request.url}", flush=True) - return c2pa.HttpResponse(cached[0], cached[1]) - print(f"[cache] MISS {request.url}", flush=True) - - status, body = self._fetch_with_retries(request) - if cacheable and status == 200: - self.cache.put(request.url, (status, body)) - return c2pa.HttpResponse(status, body) - - def _fetch_with_retries(self, request): - data = request.body or None - for attempt in range(self._max_retries + 1): - req = urllib.request.Request( - request.url, - data=data, - method=request.method, - headers=request.headers) - try: - with urllib.request.urlopen( - req, timeout=self._timeout) as resp: - return resp.status, resp.read() - except urllib.error.HTTPError as e: - retryable = e.code in (429, 503) - if not retryable or attempt == self._max_retries: - # Final failure: - # Pass the status back so the SDK can report it.. - return e.code, e.read() - delay = self._retry_delay(e, attempt) - print(f"[http] {e.code}, retrying in {delay:.1f}s", - flush=True) - time.sleep(delay) - raise RuntimeError("unreachable") - - def _retry_delay(self, error, attempt): - retry_after = error.headers.get("Retry-After") - if retry_after: - try: - return min(float(retry_after), self._max_retry_after) - except ValueError: - pass - return self._backoff * (2 ** attempt) - - -def read_with_cache(): - """Read the same remote-manifest asset more than once for cache hits.""" - print("\n--- Reading cloud.jpg multiple times should hit a cache") - - resolver = CachingHttpResolver() - with c2pa.Context.builder().with_resolver(resolver).build() as context: - for round_number in (1, 2): - with open(fixtures_dir + "cloud.jpg", "rb") as f: - with c2pa.Reader("image/jpeg", f, context=context) as reader: - print(f" read {round_number}: " - f"{reader.get_validation_state()}") - - print(f"cache hits={resolver.cache.hits} misses={resolver.cache.misses}") - - -def sign_with_cache(): - """Add the same remote-manifest ingredient repeatedly, hitting the cache. - - Each add re-reads the ingredient's remote manifest through the context - resolver, so three adds mean one network fetch and two cache hits. - """ - print("\n--- Signing with the same remote ingredient added multiple times...") - - with open(fixtures_dir + "es256_certs.pem", "rb") as f: - certs = f.read() - with open(fixtures_dir + "es256_private.key", "rb") as f: - key = f.read() - - # ta_url is None, meaning "no timestamp authority". - # An empty string is treated as a URL and fails signing. - signer_info = c2pa.C2paSignerInfo( - alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) - - ingredient_labels = [f"cloud-ingredient-{i + 1}" for i in range(3)] - - manifest_definition = { - "claim_generator": "http_resolver_cache", - "claim_generator_info": [ - {"name": "http_resolver_cache", "version": "0.1"}], - "format": "image/jpeg", - "title": "Signed with a caching HTTP resolver", - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [ - { - "action": "c2pa.created", - "digitalSourceType": - "http://cv.iptc.org/newscodes/" - "digitalsourcetype/digitalCreation", - }, - { - "action": "c2pa.placed", - "parameters": {"ingredientIds": ingredient_labels}, - }, - ]}, - }], - } - - # A fresh resolver per flow so it prints nicely. - resolver = CachingHttpResolver() - context = (c2pa.Context.builder() - .with_resolver(resolver) - .with_signer(c2pa.Signer.from_info(signer_info)) - .build()) - try: - builder = c2pa.Builder(manifest_definition, context=context) - - with open(fixtures_dir + "cloud.jpg", "rb") as ingredient: - for index in range(3): - ingredient.seek(0) - builder.add_ingredient( - {"title": f"cloud.jpg #{index + 1}", - "relationship": "componentOf", - "label": ingredient_labels[index]}, - "image/jpeg", ingredient) - - os.makedirs(output_dir, exist_ok=True) - output_path = output_dir + "A_signed_cached.jpg" - with open(fixtures_dir + "A.jpg", "rb") as source: - with open(output_path, "wb") as dest: - builder.sign( - c2pa.Signer.from_info(signer_info), - "image/jpeg", source, dest) - print(f"signed asset written to {output_path}") - finally: - context.close() - - print(f"cache hits={resolver.cache.hits} misses={resolver.cache.misses} " - "(expected 2 hits, 1 miss)") - - -def failing_resolver_is_a_clean_error(): - """A final failure surfaces as a typed error, not a crash.""" - print("\n--- A resolver that always answers 500") - - def always_500(request): - return c2pa.HttpResponse(500, b"") - - with c2pa.Context.builder().with_resolver(always_500).build() as context: - try: - with open(fixtures_dir + "cloud.jpg", "rb") as f: - c2pa.Reader("image/jpeg", f, context=context) - print("unexpected success") - except c2pa.C2paError as e: - print(f"got a typed error as expected: {e}") - - -def main(): - try: - read_with_cache() - sign_with_cache() - except c2pa.C2paError as e: - message = str(e) - if "fetch" in message or "resolver" in message: - print(f"\nError: {e}") - print("This example needs internet access to fetch the remote " - "manifest for tests/fixtures/cloud.jpg.") - sys.exit(1) - raise - - # This one needs no network: the resolver answers without calling out. - failing_resolver_is_a_clean_error() - - -if __name__ == "__main__": - main() diff --git a/examples/http_resolver_debug.py b/examples/http_resolver_debug.py deleted file mode 100644 index 3e726d09..00000000 --- a/examples/http_resolver_debug.py +++ /dev/null @@ -1,164 +0,0 @@ -import json -import os -import sys -import urllib.error -import urllib.request - -import c2pa - -# This example shows how to intercept every HTTP request the C2PA SDK makes, -# by attaching a custom HTTP resolver to a Context. -# The resolver logs the method and URL of each request and the status code -# of each response, while delegating the actual transfer to urllib from -# the standard library. The resolver sees all network traffic. - -fixtures_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "tests", "fixtures") + os.sep -output_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "output") + os.sep - - -class DebugHttpResolver: - """Logs every SDK HTTP request and response status. - The actual transfer is delegated to urllib. - """ - - def __init__(self, timeout=10.0): - self._timeout = timeout - - def resolve(self, request): - print(f"[http] {request.method} {request.url}", flush=True) - - # Timestamp requests POST a body; manifest fetches send none. - data = request.body or None - req = urllib.request.Request( - request.url, - data=data, - method=request.method, - headers=request.headers) - - try: - with urllib.request.urlopen(req, timeout=self._timeout) as resp: - body = resp.read() - print(f"[http] -> {resp.status} ({len(body)} bytes)", - flush=True) - return c2pa.HttpResponse(resp.status, body) - except urllib.error.HTTPError as e: - # A 4xx/5xx is still a response! - # Pass the status through and let the SDK turn it into - # its own typed error: a remote manifest fetch only accepts 200. - body = e.read() - print(f"[http] -> {e.code}", flush=True) - return c2pa.HttpResponse(e.code, body) - # urllib.error.URLError (DNS failure, connection refused, timeout) is deliberately not caught: - # raising marks the request as a hard resolver failure, which surfaces as a typed C2paError. - - -def read_with_resolver(): - """Read an asset whose manifest lives at a remote URL.""" - print("\n--- Reading cloud.jpg (manifest is remote):") - - resolver = DebugHttpResolver() - with c2pa.Context.builder().with_resolver(resolver).build() as context: - with open(fixtures_dir + "cloud.jpg", "rb") as f: - with c2pa.Reader("image/jpeg", f, context=context) as reader: - print(f"validation state: {reader.get_validation_state()}") - print(f"embedded manifest: {reader.is_embedded()}") - print(f"remote URL: {reader.get_remote_url()}") - - -def sign_with_resolver(): - """Sign an asset, adding an ingredient whose manifest is remote.""" - print("\n--- Signing A.jpg with a remote-manifest ingredient:") - - with open(fixtures_dir + "es256_certs.pem", "rb") as f: - certs = f.read() - with open(fixtures_dir + "es256_private.key", "rb") as f: - key = f.read() - - # ta_url is None, meaning "no timestamp authority". - signer_info = c2pa.C2paSignerInfo( - alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) - - manifest_definition = { - "claim_generator": "http_resolver_debug", - "claim_generator_info": [ - {"name": "http_resolver_debug", "version": "0.1"}], - "format": "image/jpeg", - "title": "Signed with a debug HTTP resolver", - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [ - { - "action": "c2pa.created", - "digitalSourceType": - "http://cv.iptc.org/newscodes/" - "digitalsourcetype/digitalCreation", - }, - { - "action": "c2pa.placed", - "parameters": {"ingredientIds": ["cloud-ingredient"]}, - }, - ]}, - }], - } - - resolver = DebugHttpResolver() - context = (c2pa.Context.builder() - .with_resolver(resolver) - .with_signer(c2pa.Signer.from_info(signer_info)) - .build()) - try: - builder = c2pa.Builder(manifest_definition, context=context) - - # Adding this ingredient fetches its remote manifest through the resolver, - # so one GET shows up in the log. - with open(fixtures_dir + "cloud.jpg", "rb") as ingredient: - builder.add_ingredient( - {"title": "cloud.jpg", "relationship": "componentOf", - "label": "cloud-ingredient"}, - "image/jpeg", ingredient) - - os.makedirs(output_dir, exist_ok=True) - output_path = output_dir + "A_signed_resolver.jpg" - with open(fixtures_dir + "A.jpg", "rb") as source: - with open(output_path, "wb") as dest: - builder.sign( - c2pa.Signer.from_info(signer_info), - "image/jpeg", source, dest) - print(f"signed asset written to {output_path}") - - # Reading the signed file back uses its embedded manifest, - # so this makes no HTTP requests at all. - print("\n--- Reading the signed asset (no HTTP expected): ") - with open(output_path, "rb") as f: - with c2pa.Reader("image/jpeg", f) as reader: - store = json.loads(reader.json()) - manifest = store["manifests"][store["active_manifest"]] - for ingredient in manifest.get("ingredients", []): - print(f"ingredient: {ingredient.get('title')}") - finally: - context.close() - - -def main(): - try: - read_with_resolver() - sign_with_resolver() - except c2pa.C2paError as e: - message = str(e) - if "fetch" in message or "resolver" in message: - print(f"\nError: {e}") - print("This example needs internet access to fetch the remote " - "manifest for tests/fixtures/cloud.jpg.") - sys.exit(1) - raise - - print("\nNote: with a ta_url set on the signer, the log above would also " - "show the RFC 3161 timestamp POST.") - - -if __name__ == "__main__": - main() diff --git a/src/c2pa/__init__.py b/src/c2pa/__init__.py index 9c7f6bff..bd581d58 100644 --- a/src/c2pa/__init__.py +++ b/src/c2pa/__init__.py @@ -31,8 +31,6 @@ Context, ContextBuilder, ContextProvider, - HttpRequest, - HttpResponse, sdk_version, load_settings ) # NOQA @@ -52,8 +50,6 @@ 'Context', 'ContextBuilder', 'ContextProvider', - 'HttpRequest', - 'HttpResponse', 'sdk_version', 'load_settings' ] diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index a03e10d9..d7fc9ddc 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -650,50 +650,6 @@ def __del__(self): ctypes.c_ubyte), ctypes.c_size_t) -class C2paHttpRequest(ctypes.Structure): - """Mirror of the native C2paHttpRequest (#[repr(C)]). - - Read-only view: every pointer borrows native memory that is only valid - for the duration of the resolver callback. Copy anything you keep. - - The string fields are c_char_p so ctypes converts them to bytes on read. - `body` stays a c_ubyte pointer because it is binary, not NUL-terminated. - """ - _fields_ = [ - ("url", ctypes.c_char_p), - ("method", ctypes.c_char_p), - ("headers", ctypes.c_char_p), - ("body", ctypes.POINTER(ctypes.c_ubyte)), - ("body_len", ctypes.c_size_t), - ] - - -class C2paHttpResponse(ctypes.Structure): - """Mirror of the native C2paHttpResponse (#[repr(C)]). - - The callback fills this in. `body` must be allocated with the C runtime - malloc that the native library's free() matches: the native side takes - ownership and frees it on both the success and the error return path. - """ - _fields_ = [ - ("status", ctypes.c_int), - ("body", ctypes.POINTER(ctypes.c_ubyte)), - ("body_len", ctypes.c_size_t), - ] - - -class C2paHttpResolver(ctypes.Structure): - """Opaque structure for a native HTTP resolver handle.""" - _fields_ = [] # Empty as it's opaque in the C API - - -HttpResolverCallback = ctypes.CFUNCTYPE( - ctypes.c_int, - ctypes.c_void_p, - ctypes.POINTER(C2paHttpRequest), - ctypes.POINTER(C2paHttpResponse)) - - class StreamContext(ctypes.Structure): """Opaque structure for stream context.""" _fields_ = [] # Empty as it's opaque in the C API @@ -1090,17 +1046,6 @@ def _setup_function(func, argtypes, restype=None): ctypes.c_int ) -# HTTP resolver bindings -_setup_function( - _lib.c2pa_http_resolver_create, - [ctypes.c_void_p, HttpResolverCallback], - ctypes.POINTER(C2paHttpResolver) -) -_setup_function( - _lib.c2pa_context_builder_set_http_resolver, - [ctypes.POINTER(C2paContextBuilder), ctypes.POINTER(C2paHttpResolver)], - ctypes.c_int -) _setup_function( _lib.c2pa_error_set_last, [ctypes.c_char_p], @@ -1559,194 +1504,284 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: return mimetypes.guess_type(str(path))[0] or "" -_NATIVE_MALLOC = None +class NetworkResource(ManagedResource): + """Owns a native HTTP resolver handle for a Context. + Private implementation detail of the custom-resolver feature: not part + of the public API. Nests every native mirror/helper the feature needs + (ctypes structs, the malloc used for response bodies, header parsing, + resolver normalization, the ctypes trampoline builder, and the request/ + response data shapes) so nothing about this feature is part of the + public surface. Context is the only thing that uses this class. -def _get_native_malloc(): - """Return malloc from the C runtime whose free() the native library calls. + A custom resolver never needs to import HttpRequest/HttpResponse: it + receives a request-like object (.url, .method, .headers, .body) and + may return any response-like object (.status, .body) -- duck typing, + not isinstance checks, is what _make_trampoline relies on below. + """ - Resolver response bodies are handed to the native library, which frees - them with the platform libc free(). The allocation must come from the - matching runtime or the free is heap corruption, not a leak. + class _Request(ctypes.Structure): + """Mirror of the native C2paHttpRequest (#[repr(C)]). - Looked up lazily so importing c2pa never fails on an exotic platform - unless the resolver feature is actually used. - """ - global _NATIVE_MALLOC - if _NATIVE_MALLOC is None: - if sys.platform == "win32": - try: - # Rust MSVC targets link the UCRT, so libc::free is - # ucrtbase!free. The legacy msvcrt.dll is a different heap. - crt = ctypes.CDLL("ucrtbase") - except OSError: - crt = ctypes.CDLL("msvcrt") - else: - crt = ctypes.CDLL(None) - malloc = crt.malloc - malloc.argtypes = [ctypes.c_size_t] - # Required: the default c_int restype truncates 64-bit pointers. - malloc.restype = ctypes.c_void_p - _NATIVE_MALLOC = malloc - return _NATIVE_MALLOC + Read-only view: every pointer borrows native memory that is only + valid for the duration of the resolver callback. Copy anything you + keep. + The string fields are c_char_p so ctypes converts them to bytes on + read. `body` stays a c_ubyte pointer because it is binary, not + NUL-terminated. + """ + _fields_ = [ + ("url", ctypes.c_char_p), + ("method", ctypes.c_char_p), + ("headers", ctypes.c_char_p), + ("body", ctypes.POINTER(ctypes.c_ubyte)), + ("body_len", ctypes.c_size_t), + ] + + class _Response(ctypes.Structure): + """Mirror of the native C2paHttpResponse (#[repr(C)]). + + The callback fills this in. `body` must be allocated with the C + runtime malloc that the native library's free() matches: the + native side takes ownership and frees it on both the success and + the error return path. + """ + _fields_ = [ + ("status", ctypes.c_int), + ("body", ctypes.POINTER(ctypes.c_ubyte)), + ("body_len", ctypes.c_size_t), + ] + + class _Handle(ctypes.Structure): + """Opaque structure for a native HTTP resolver handle.""" + _fields_ = [] # Empty as it's opaque in the C API + + _Callback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_void_p, + ctypes.POINTER(_Request), + ctypes.POINTER(_Response)) + + class HttpRequest: + """An HTTP request the SDK asks a custom resolver to perform. + + Attributes: + url: Absolute request URL. + method: HTTP method ("GET", "POST", ...). + headers: Request headers as a dict. Names are lowercased by the + native layer; when a header repeats, the last value wins. + body: Request body bytes (b"" when there is none). Timestamp + requests POST a body; manifest fetches send none. + + All data is copied out of native memory, so it stays valid after + the resolver call returns. Not a public type: a resolver reads + attributes off the instance it's handed, it never imports this + class. + """ + __slots__ = ("url", "method", "headers", "body") -def _parse_header_lines(raw: str) -> dict: - """Parse the FFI's newline-delimited 'Name: Value' header block. + def __init__( + self, url: str, method: str, headers: dict, body: bytes): + self.url = url + self.method = method + self.headers = headers + self.body = body - The native side always sends a string (empty when there are no headers), - never NULL. Header names arrive lowercased, and repeated headers are sent - as separate lines, so the last occurrence of a name wins here. - """ - headers = {} - for line in raw.split("\n"): - name, sep, value = line.partition(":") - if sep: - headers[name.strip()] = value.strip() - return headers - - -class HttpRequest: - """An HTTP request the SDK asks a custom resolver to perform. - - Attributes: - url: Absolute request URL. - method: HTTP method ("GET", "POST", ...). - headers: Request headers as a dict. Names are lowercased by the - native layer; when a header repeats, the last value wins. - body: Request body bytes (b"" when there is none). Timestamp - requests POST a body; manifest fetches send none. - - All data is copied out of native memory, so it stays valid after the - resolver call returns. - """ - __slots__ = ("url", "method", "headers", "body") + def __repr__(self): + return (f"HttpRequest(method={self.method!r}, " + f"url={self.url!r}, body_len={len(self.body)})") - def __init__(self, url: str, method: str, headers: dict, body: bytes): - self.url = url - self.method = method - self.headers = headers - self.body = body + class HttpResponse: + """The answer a custom resolver returns to the SDK. - def __repr__(self): - return (f"HttpRequest(method={self.method!r}, url={self.url!r}, " - f"body_len={len(self.body)})") + Attributes: + status: HTTP status code. Remote manifest fetches only accept + 200; any other code surfaces as a typed C2paError. + body: Response body bytes. + Not a public type: a resolver may return any object exposing these + two attributes, it never needs to import this class. + """ + __slots__ = ("status", "body") -class HttpResponse: - """The answer a custom resolver returns to the SDK. + def __init__(self, status: int, body: bytes = b""): + self.status = status + self.body = body - Attributes: - status: HTTP status code. Remote manifest fetches only accept 200; - any other code surfaces as a typed C2paError. - body: Response body bytes. - """ - __slots__ = ("status", "body") + def __repr__(self): + return (f"HttpResponse(status={self.status}, " + f"body_len={len(self.body or b'')})") - def __init__(self, status: int, body: bytes = b""): - self.status = status - self.body = body + _native_malloc = None - def __repr__(self): - return (f"HttpResponse(status={self.status}, " - f"body_len={len(self.body or b'')})") + @staticmethod + def _get_native_malloc(): + """Return malloc from the C runtime whose free() the native library + calls. + Resolver response bodies are handed to the native library, which + frees them with the platform libc free(). The allocation must come + from the matching runtime or the free is heap corruption, not a + leak. -def _coerce_resolver(resolver): - """Normalize a resolver into a plain callable taking an HttpRequest. + Looked up lazily so importing c2pa never fails on an exotic + platform unless the resolver feature is actually used. + """ + if NetworkResource._native_malloc is None: + if sys.platform == "win32": + try: + # Rust MSVC targets link the UCRT, so libc::free is + # ucrtbase!free. The legacy msvcrt.dll is a different + # heap. + crt = ctypes.CDLL("ucrtbase") + except OSError: + crt = ctypes.CDLL("msvcrt") + else: + crt = ctypes.CDLL(None) + malloc = crt.malloc + malloc.argtypes = [ctypes.c_size_t] + # Required: the default c_int restype truncates 64-bit pointers. + malloc.restype = ctypes.c_void_p + NetworkResource._native_malloc = malloc + return NetworkResource._native_malloc - Accepts either an object with a resolve(request) method or a bare - callable with the same signature. - """ - resolve_fn = getattr(resolver, "resolve", None) - if callable(resolve_fn): - return resolve_fn - if callable(resolver): - return resolver - raise C2paError( - "HTTP resolver must be callable or provide a resolve() method") + @staticmethod + def _parse_header_lines(raw: str) -> dict: + """Parse the FFI's newline-delimited 'Name: Value' header block. + The native side always sends a string (empty when there are no + headers), never NULL. Header names arrive lowercased, and repeated + headers are sent as separate lines, so the last occurrence of a + name wins here. + """ + headers = {} + for line in raw.split("\n"): + name, sep, value = line.partition(":") + if sep: + headers[name.strip()] = value.strip() + return headers -def _make_http_resolver_trampoline(resolve_fn): - """Wrap a Python resolve function into a native C callback. + @staticmethod + def _coerce_resolver(resolver): + """Normalize a resolver into a plain callable taking an HttpRequest. - Args: - resolve_fn: Callable[[HttpRequest], HttpResponse]. + Accepts either an object with a resolve(request) method or a bare + callable with the same signature. + """ + resolve_fn = getattr(resolver, "resolve", None) + if callable(resolve_fn): + return resolve_fn + if callable(resolver): + return resolver + raise C2paError( + "HTTP resolver must be callable or provide a resolve() method") - Returns: - The HttpResolverCallback object. The caller MUST keep a reference to - it for as long as any native context built from it can run: the - native side holds only a raw function pointer, and letting the thunk - be collected while a context can still call it is undefined behavior. - """ - def _trampoline(_ctx, request_ptr, response_ptr): - try: - req = request_ptr.contents - # Copy everything out now: these pointers borrow native memory - # that is only valid for the duration of this call. - url = req.url.decode("utf-8", "replace") if req.url else "" - method = (req.method.decode("utf-8", "replace") - if req.method else "") - raw_headers = (req.headers.decode("utf-8", "replace") - if req.headers else "") - body = (ctypes.string_at(req.body, req.body_len) - if (req.body and req.body_len) else b"") - - result = resolve_fn(HttpRequest( - url=url, - method=method, - headers=_parse_header_lines(raw_headers), - body=body)) - - payload = result.body or b"" - if not isinstance(payload, (bytes, bytearray)): - raise TypeError( - "HttpResponse.body must be bytes, got " - f"{type(payload).__name__}") - - response = response_ptr.contents - response.status = int(result.status) - if payload: - length = len(payload) - buf = _get_native_malloc()(length) - if not buf: - _lib.c2pa_error_set_last( - b"Other: HTTP resolver out of memory") - return -1 - ctypes.memmove(buf, bytes(payload), length) - # Ownership handoff: from here the native library frees this - # buffer on BOTH return paths (it copies then frees on 0, and - # frees a leftover body on non-zero). Never free it here. - response.body = ctypes.cast( - buf, ctypes.POINTER(ctypes.c_ubyte)) - response.body_len = length - else: - # body and body_len must stay NULL/0 together: the native - # side skips its free when body_len is 0, so a non-NULL - # pointer with a zero length is never freed and leaks. - response.body = None - response.body_len = 0 - return 0 - except BaseException as e: # noqa: B036 - must not unwind into native - # BaseException on purpose: an exception escaping a ctypes - # callback cannot propagate into Rust, so it would be reported - # as a generic failure with the real cause lost. Catching it - # here (including KeyboardInterrupt) turns it into a typed error - # carrying the actual message. - # - # Setting the error is mandatory, not best-effort: the native - # error slot is thread-local and is NOT cleared before the - # callback runs, so returning non-zero without setting it - # surfaces a stale, unrelated error from an earlier call. + @classmethod + def _make_trampoline(cls, resolve_fn): + """Wrap a Python resolve function into a native C callback. + + Args: + resolve_fn: Callable[[HttpRequest], HttpResponse]. + + Returns: + The _Callback object. The caller MUST keep a reference to it + for as long as any native context built from it can run: the + native side holds only a raw function pointer, and letting the + thunk be collected while a context can still call it is + undefined behavior. + """ + def _trampoline(_ctx, request_ptr, response_ptr): try: - _lib.c2pa_error_set_last( - "Other: Python HTTP resolver failed: {}".format(e) - .encode("utf-8", "replace")) - except BaseException: - pass - return -1 + req = request_ptr.contents + # Copy everything out now: these pointers borrow native + # memory that is only valid for the duration of this call. + url = req.url.decode("utf-8", "replace") if req.url else "" + method = (req.method.decode("utf-8", "replace") + if req.method else "") + raw_headers = (req.headers.decode("utf-8", "replace") + if req.headers else "") + body = (ctypes.string_at(req.body, req.body_len) + if (req.body and req.body_len) else b"") + + result = resolve_fn(cls.HttpRequest( + url=url, + method=method, + headers=cls._parse_header_lines(raw_headers), + body=body)) + + payload = result.body or b"" + if not isinstance(payload, (bytes, bytearray)): + raise TypeError( + "HttpResponse.body must be bytes, got " + f"{type(payload).__name__}") + + response = response_ptr.contents + response.status = int(result.status) + if payload: + length = len(payload) + buf = cls._get_native_malloc()(length) + if not buf: + _lib.c2pa_error_set_last( + b"Other: HTTP resolver out of memory") + return -1 + ctypes.memmove(buf, bytes(payload), length) + # Ownership handoff: from here the native library frees + # this buffer on BOTH return paths (it copies then + # frees on 0, and frees a leftover body on non-zero). + # Never free it here. + response.body = ctypes.cast( + buf, ctypes.POINTER(ctypes.c_ubyte)) + response.body_len = length + else: + # body and body_len must stay NULL/0 together: the + # native side skips its free when body_len is 0, so a + # non-NULL pointer with a zero length is never freed + # and leaks. + response.body = None + response.body_len = 0 + return 0 + except BaseException as e: # noqa: B036 - must not unwind native + # BaseException on purpose: an exception escaping a ctypes + # callback cannot propagate into Rust, so it would be + # reported as a generic failure with the real cause lost. + # Catching it here (including KeyboardInterrupt) turns it + # into a typed error carrying the actual message. + # + # Setting the error is mandatory, not best-effort: the + # native error slot is thread-local and is NOT cleared + # before the callback runs, so returning non-zero without + # setting it surfaces a stale, unrelated error from an + # earlier call. + try: + _lib.c2pa_error_set_last( + "Other: Python HTTP resolver failed: {}".format(e) + .encode("utf-8", "replace")) + except BaseException: + pass + return -1 + + return cls._Callback(_trampoline) - return HttpResolverCallback(_trampoline) + def __init__(self, callback_cb): + super().__init__() + self._create_and_activate( + lambda: _lib.c2pa_http_resolver_create(None, callback_cb), + "Failed to create HTTP resolver") + + +# HTTP resolver bindings +_setup_function( + _lib.c2pa_http_resolver_create, + [ctypes.c_void_p, NetworkResource._Callback], + ctypes.POINTER(NetworkResource._Handle) +) +_setup_function( + _lib.c2pa_context_builder_set_http_resolver, + [ctypes.POINTER(C2paContextBuilder), + ctypes.POINTER(NetworkResource._Handle)], + ctypes.c_int +) class ContextProvider(ABC): @@ -1929,11 +1964,14 @@ def with_resolver( Can be called multiple times; the last resolver wins. Args: - resolver: An object with a resolve(request) -> HttpResponse - method, or a callable with the same signature. It receives - an HttpRequest and must return an HttpResponse. Raising - instead marks the request as a hard failure, which surfaces - as a typed C2paError. + resolver: An object with a resolve(request) method, or a + callable with the same signature. It receives a request + object exposing .url (str), .method (str), .headers + (dict), and .body (bytes), and must return a response + object exposing .status (int) and .body (bytes) -- any + object with those attributes works, there is no type to + import. Raising instead marks the request as a hard + failure, which surfaces as a typed C2paError. Returns: self, for method chaining. @@ -1997,19 +2035,6 @@ def __init__(self): _lib.c2pa_context_builder_new, "Failed to create ContextBuilder") - class _NativeHttpResolver(ManagedResource): - """Short-lived wrapper for a native C2paHttpResolver handle. - - Any failure inside its `with` block frees it via close(), unless a - consuming call already took ownership of it. - """ - - def __init__(self, callback_cb): - super().__init__() - self._create_and_activate( - lambda: _lib.c2pa_http_resolver_create(None, callback_cb), - "Failed to create HTTP resolver") - def __init__( self, settings: Optional['Settings'] = None, @@ -2024,9 +2049,9 @@ def __init__( signer: Optional Signer. If provided it is consumed and must not be used directly again after that call. resolver: Optional custom HTTP resolver, either an object with a - resolve(request) -> HttpResponse method or a callable with - the same signature. See ContextBuilder.with_resolver for the - full contract. + resolve(request) method or a callable with the same + signature. See ContextBuilder.with_resolver for the full + contract. Raises: C2paError: If creation fails @@ -2051,12 +2076,13 @@ def __init__( check=lambda r: r != 0) if resolver is not None: - resolve_fn = _coerce_resolver(resolver) - callback_cb = _make_http_resolver_trampoline(resolve_fn) + resolve_fn = NetworkResource._coerce_resolver(resolver) + callback_cb = NetworkResource._make_trampoline( + resolve_fn) # Pin the thunk before any native call can capture its # raw function pointer (see _release for why it stays). self._http_resolver_cb = callback_cb - with self._NativeHttpResolver(callback_cb) as native_res: + with NetworkResource(callback_cb) as native_res: native_res._consume_no_replacement( lambda h: _lib.c2pa_context_builder_set_http_resolver( diff --git a/tests/network/__init__.py b/tests/network/__init__.py new file mode 100644 index 00000000..dcf2c804 --- /dev/null +++ b/tests/network/__init__.py @@ -0,0 +1 @@ +# Placeholder diff --git a/tests/network/test_http_resolver_cache.py b/tests/network/test_http_resolver_cache.py new file mode 100644 index 00000000..2ad9f7c6 --- /dev/null +++ b/tests/network/test_http_resolver_cache.py @@ -0,0 +1,283 @@ +# Copyright 2025 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, +# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) +# or the MIT license (http://opensource.org/licenses/MIT), +# at your option. + +# Unless required by applicable law or agreed to in writing, +# this software is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or +# implied. See the LICENSE-MIT and LICENSE-APACHE files for the +# specific language governing permissions and limitations under +# each license. + +"""A custom HTTP resolver that caches responses and retries throttled +requests, exercised against the real network. + +Ported from the former examples/http_resolver_cache.py: still a runnable +demonstration of the resolver pattern, now also verified in CI. Needs +internet access to fetch the remote manifest for tests/fixtures/cloud.jpg; +tests skip (not fail) when that fetch can't reach the network. +""" + +import collections +import os +import sys +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +from typing import NamedTuple + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__)))) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + +import c2pa # noqa: E402 + +FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir, "fixtures") + + +class HttpResponse(NamedTuple): + """Duck-typed stand-in for c2pa's internal resolver response shape. + + The resolver contract only requires .status (int) and .body (bytes); + there is no public type to import, any object with those attributes + works. + """ + status: int + body: bytes = b"" + + +def _skip_if_offline(testcase, exc): + """Skip the test if exc looks like a network-reachability failure. + + Re-raises anything else, so a real bug in the resolver or the SDK + still fails the test instead of being silently skipped. + """ + message = str(exc) + if "fetch" in message or "resolver" in message: + testcase.skipTest( + "needs internet access to fetch the remote manifest for " + f"tests/fixtures/cloud.jpg: {exc}") + raise exc + + +class TtlLruCache: + """A small LRU cache whose entries also expire after a TTL.""" + + def __init__(self, max_items=100, ttl_seconds=120.0): + self._max_items = int(max_items) + self._ttl = float(ttl_seconds) + self._entries = collections.OrderedDict() # key -> (expiry, value) + self._lock = threading.Lock() + self.hits = 0 + self.misses = 0 + + def get(self, key): + with self._lock: + entry = self._entries.get(key) + # time.monotonic, not time.time: immune to wall-clock jumps. + if entry is not None and entry[0] > time.monotonic(): + self._entries.move_to_end(key) + self.hits += 1 + return entry[1] + if entry is not None: + del self._entries[key] + self.misses += 1 + return None + + def put(self, key, value): + with self._lock: + if key in self._entries: + self._entries.move_to_end(key) + self._entries[key] = (time.monotonic() + self._ttl, value) + while len(self._entries) > self._max_items: + self._entries.popitem(last=False) + + +class CachingHttpResolver: + """An HTTP resolver with a response cache and bounded retries. + + Caching policy: only read GET requests answered with 200 are cached. + POSTs (timestamp requests) and error responses are not. + + Retry policy: 429 and 503 are retried up to max_retries times, + honoring a capped Retry-After when the header is present, + and otherwise backing off exponentially. + Any other status is final and is passed through to the SDK. + Transport errors raise, which marks a hard failure. + """ + + def __init__(self, cache=None, timeout=10.0, max_retries=3, + backoff_seconds=0.5, max_retry_after=10.0): + self.cache = cache if cache is not None else TtlLruCache() + self._timeout = timeout + self._max_retries = int(max_retries) + self._backoff = float(backoff_seconds) + self._max_retry_after = float(max_retry_after) + + def resolve(self, request): + cacheable = request.method.upper() == "GET" + if cacheable: + cached = self.cache.get(request.url) + if cached is not None: + return HttpResponse(cached[0], cached[1]) + + status, body = self._fetch_with_retries(request) + if cacheable and status == 200: + self.cache.put(request.url, (status, body)) + return HttpResponse(status, body) + + def _fetch_with_retries(self, request): + data = request.body or None + for attempt in range(self._max_retries + 1): + req = urllib.request.Request( + request.url, + data=data, + method=request.method, + headers=request.headers) + try: + with urllib.request.urlopen( + req, timeout=self._timeout) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + retryable = e.code in (429, 503) + if not retryable or attempt == self._max_retries: + # Final failure: pass the status back so the SDK can + # report it. + return e.code, e.read() + delay = self._retry_delay(e, attempt) + time.sleep(delay) + raise RuntimeError("unreachable") + + def _retry_delay(self, error, attempt): + retry_after = error.headers.get("Retry-After") + if retry_after: + try: + return min(float(retry_after), self._max_retry_after) + except ValueError: + pass + return self._backoff * (2 ** attempt) + + +class TestHttpResolverCache(unittest.TestCase): + + def test_read_with_cache(self): + """Reading the same remote-manifest asset twice hits the cache.""" + resolver = CachingHttpResolver() + try: + with c2pa.Context.builder().with_resolver( + resolver).build() as context: + for _ in range(2): + with open(os.path.join(FIXTURES, "cloud.jpg"), + "rb") as f: + with c2pa.Reader( + "image/jpeg", f, context=context) as reader: + reader.get_validation_state() + except c2pa.C2paError as e: + _skip_if_offline(self, e) + + self.assertEqual(resolver.cache.hits, 1) + self.assertEqual(resolver.cache.misses, 1) + + def test_sign_with_cache(self): + """Adding the same remote-manifest ingredient repeatedly hits the + cache. + + Each add re-reads the ingredient's remote manifest through the + context resolver, so three adds mean one network fetch and two + cache hits. + """ + with open(os.path.join(FIXTURES, "es256_certs.pem"), "rb") as f: + certs = f.read() + with open(os.path.join(FIXTURES, "es256_private.key"), "rb") as f: + key = f.read() + + # ta_url is None, meaning "no timestamp authority". + # An empty string is treated as a URL and fails signing. + signer_info = c2pa.C2paSignerInfo( + alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) + + ingredient_labels = [f"cloud-ingredient-{i + 1}" for i in range(3)] + + manifest_definition = { + "claim_generator": "http_resolver_cache", + "claim_generator_info": [ + {"name": "http_resolver_cache", "version": "0.1"}], + "format": "image/jpeg", + "title": "Signed with a caching HTTP resolver", + "assertions": [{ + "label": "c2pa.actions.v2", + "data": {"actions": [ + { + "action": "c2pa.created", + "digitalSourceType": + "http://cv.iptc.org/newscodes/" + "digitalsourcetype/digitalCreation", + }, + { + "action": "c2pa.placed", + "parameters": {"ingredientIds": ingredient_labels}, + }, + ]}, + }], + } + + resolver = CachingHttpResolver() + context = (c2pa.Context.builder() + .with_resolver(resolver) + .with_signer(c2pa.Signer.from_info(signer_info)) + .build()) + try: + try: + builder = c2pa.Builder(manifest_definition, context=context) + + with open(os.path.join(FIXTURES, "cloud.jpg"), + "rb") as ingredient: + for index in range(3): + ingredient.seek(0) + builder.add_ingredient( + {"title": f"cloud.jpg #{index + 1}", + "relationship": "componentOf", + "label": ingredient_labels[index]}, + "image/jpeg", ingredient) + + with tempfile.TemporaryDirectory() as output_dir: + output_path = os.path.join( + output_dir, "A_signed_cached.jpg") + with open(os.path.join(FIXTURES, "A.jpg"), + "rb") as source: + with open(output_path, "wb") as dest: + builder.sign( + c2pa.Signer.from_info(signer_info), + "image/jpeg", source, dest) + self.assertTrue(os.path.exists(output_path)) + except c2pa.C2paError as e: + _skip_if_offline(self, e) + finally: + context.close() + + self.assertEqual(resolver.cache.misses, 1) + self.assertEqual(resolver.cache.hits, 2) + + def test_failing_resolver_is_a_clean_error(self): + """A final failure surfaces as a typed error, not a crash. + + Needs no network: the resolver answers without calling out. + """ + def always_500(request): + return HttpResponse(500, b"") + + with c2pa.Context.builder().with_resolver( + always_500).build() as context: + with self.assertRaises(c2pa.C2paError): + with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: + c2pa.Reader("image/jpeg", f, context=context) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/network/test_http_resolver_debug.py b/tests/network/test_http_resolver_debug.py new file mode 100644 index 00000000..adc8dff6 --- /dev/null +++ b/tests/network/test_http_resolver_debug.py @@ -0,0 +1,206 @@ +# Copyright 2025 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, +# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) +# or the MIT license (http://opensource.org/licenses/MIT), +# at your option. + +# Unless required by applicable law or agreed to in writing, +# this software is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or +# implied. See the LICENSE-MIT and LICENSE-APACHE files for the +# specific language governing permissions and limitations under +# each license. + +"""A custom HTTP resolver that logs every request/response, exercised +against the real network. + +Ported from the former examples/http_resolver_debug.py: still a runnable +demonstration of intercepting every HTTP request the SDK makes through a +Context, now also verified in CI. Needs internet access to fetch the +remote manifest for tests/fixtures/cloud.jpg; tests skip (not fail) when +that fetch can't reach the network. +""" + +import json +import os +import sys +import tempfile +import unittest +import urllib.error +import urllib.request +from typing import NamedTuple + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__)))) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + +import c2pa # noqa: E402 + +FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir, "fixtures") + + +class HttpResponse(NamedTuple): + """Duck-typed stand-in for c2pa's internal resolver response shape. + + The resolver contract only requires .status (int) and .body (bytes); + there is no public type to import, any object with those attributes + works. + """ + status: int + body: bytes = b"" + + +def _skip_if_offline(testcase, exc): + """Skip the test if exc looks like a network-reachability failure. + + Re-raises anything else, so a real bug in the resolver or the SDK + still fails the test instead of being silently skipped. + """ + message = str(exc) + if "fetch" in message or "resolver" in message: + testcase.skipTest( + "needs internet access to fetch the remote manifest for " + f"tests/fixtures/cloud.jpg: {exc}") + raise exc + + +class DebugHttpResolver: + """Logs every SDK HTTP request and response status. + The actual transfer is delegated to urllib. + """ + + def __init__(self, timeout=10.0): + self._timeout = timeout + self.requests = [] + + def resolve(self, request): + self.requests.append((request.method, request.url)) + + # Timestamp requests POST a body; manifest fetches send none. + data = request.body or None + req = urllib.request.Request( + request.url, + data=data, + method=request.method, + headers=request.headers) + + try: + with urllib.request.urlopen(req, timeout=self._timeout) as resp: + return HttpResponse(resp.status, resp.read()) + except urllib.error.HTTPError as e: + # A 4xx/5xx is still a response! Pass the status through and + # let the SDK turn it into its own typed error: a remote + # manifest fetch only accepts 200. + return HttpResponse(e.code, e.read()) + # urllib.error.URLError (DNS failure, connection refused, timeout) + # is deliberately not caught: raising marks the request as a hard + # resolver failure, which surfaces as a typed C2paError. + + +class TestHttpResolverDebug(unittest.TestCase): + + def test_read_with_resolver(self): + """Reading an asset whose manifest lives at a remote URL logs a + GET for that fetch.""" + resolver = DebugHttpResolver() + try: + with c2pa.Context.builder().with_resolver( + resolver).build() as context: + with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: + with c2pa.Reader( + "image/jpeg", f, context=context) as reader: + reader.get_validation_state() + self.assertFalse(reader.is_embedded()) + self.assertTrue(reader.get_remote_url()) + except c2pa.C2paError as e: + _skip_if_offline(self, e) + + self.assertTrue( + any(method == "GET" for method, _ in resolver.requests)) + + def test_sign_with_resolver(self): + """Signing an asset with a remote-manifest ingredient logs a GET + for the ingredient's manifest, and reading the signed result back + makes no HTTP requests at all (its manifest is embedded).""" + with open(os.path.join(FIXTURES, "es256_certs.pem"), "rb") as f: + certs = f.read() + with open(os.path.join(FIXTURES, "es256_private.key"), "rb") as f: + key = f.read() + + # ta_url is None, meaning "no timestamp authority". + signer_info = c2pa.C2paSignerInfo( + alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) + + manifest_definition = { + "claim_generator": "http_resolver_debug", + "claim_generator_info": [ + {"name": "http_resolver_debug", "version": "0.1"}], + "format": "image/jpeg", + "title": "Signed with a debug HTTP resolver", + "assertions": [{ + "label": "c2pa.actions.v2", + "data": {"actions": [ + { + "action": "c2pa.created", + "digitalSourceType": + "http://cv.iptc.org/newscodes/" + "digitalsourcetype/digitalCreation", + }, + { + "action": "c2pa.placed", + "parameters": {"ingredientIds": ["cloud-ingredient"]}, + }, + ]}, + }], + } + + resolver = DebugHttpResolver() + context = (c2pa.Context.builder() + .with_resolver(resolver) + .with_signer(c2pa.Signer.from_info(signer_info)) + .build()) + try: + try: + builder = c2pa.Builder(manifest_definition, context=context) + + with open(os.path.join(FIXTURES, "cloud.jpg"), + "rb") as ingredient: + builder.add_ingredient( + {"title": "cloud.jpg", "relationship": "componentOf", + "label": "cloud-ingredient"}, + "image/jpeg", ingredient) + + with tempfile.TemporaryDirectory() as output_dir: + output_path = os.path.join( + output_dir, "A_signed_resolver.jpg") + with open(os.path.join(FIXTURES, "A.jpg"), + "rb") as source: + with open(output_path, "wb") as dest: + builder.sign( + c2pa.Signer.from_info(signer_info), + "image/jpeg", source, dest) + + self.assertTrue( + any(m == "GET" for m, _ in resolver.requests)) + requests_before_reread = len(resolver.requests) + + # Reading the signed file back uses its embedded + # manifest, so this makes no HTTP requests at all. + with open(output_path, "rb") as f: + with c2pa.Reader("image/jpeg", f) as reader: + store = json.loads(reader.json()) + manifest = store["manifests"][ + store["active_manifest"]] + self.assertTrue(manifest.get("ingredients")) + + self.assertEqual( + len(resolver.requests), requests_before_reread) + except c2pa.C2paError as e: + _skip_if_offline(self, e) + finally: + context.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_http_resolver.py b/tests/test_http_resolver.py deleted file mode 100644 index 098b33a9..00000000 --- a/tests/test_http_resolver.py +++ /dev/null @@ -1,553 +0,0 @@ -# Copyright 2025 Adobe. All rights reserved. -# This file is licensed to you under the Apache License, -# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) -# or the MIT license (http://opensource.org/licenses/MIT), -# at your option. - -# Unless required by applicable law or agreed to in writing, -# this software is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or -# implied. See the LICENSE-MIT and LICENSE-APACHE files for the -# specific language governing permissions and limitations under -# each license. - -"""Lifetime and memory tests for the custom HTTP resolver bindings. - -These tests are fully offline. The fixture is built once by signing a real -asset with set_no_embed() + set_remote_url(), which makes the SDK fetch the -manifest over HTTP; the test resolvers then serve those bytes from memory. - -Assertion discipline: never assert on exact native error text, because the -wording drifts between native releases. Assert only on the C2paError -(sub)type, the numeric status code, or a substring this test injected -itself. Likewise the signed fixture uses a test certificate that is not in -any trust list, so validation_state is Invalid by design: assert on the -presence of an active manifest, not on validity. -""" - -import ctypes -import gc -import io -import json -import os -import sys -import threading -import unittest - -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) - -from c2pa import ( # noqa: E402 - Builder, - C2paError, - C2paSignerInfo, - Context, - HttpResponse, - Reader, - Signer, -) -from c2pa.c2pa import ( # noqa: E402 - HttpResolverCallback, - _get_native_malloc, - _lib, - _parse_header_lines, -) - -FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") -REMOTE_URL = "http://manifests.example.test/m1.c2pa" - -MANIFEST_DEFINITION = { - "claim_generator": "python_test", - "claim_generator_info": [{"name": "python_test", "version": "0.1"}], - "format": "image/jpeg", - "title": "resolver test", - "assertions": [], -} - - -def _free_native_buffer(body_ptr): - """Free a buffer from _get_native_malloc that nothing else took over. - - Only for buffers the SDK never received: once a response body is handed - to the native layer, it owns it on every return path. - """ - crt = ctypes.CDLL("ucrtbase") if sys.platform == "win32" \ - else ctypes.CDLL(None) - crt.free.argtypes = [ctypes.c_void_p] - crt.free.restype = None - crt.free(ctypes.cast(body_ptr, ctypes.c_void_p)) - - -def _make_signer(): - """Build an es256 signer. - - ta_url is None, meaning "no timestamp authority": an empty string is - treated as a URL and fails signing with a Signature error. - """ - with open(os.path.join(FIXTURES, "es256_certs.pem"), "rb") as f: - certs = f.read() - with open(os.path.join(FIXTURES, "es256_private.key"), "rb") as f: - key = f.read() - return Signer.from_info(C2paSignerInfo(b"es256", certs, key, None)) - - -class CountingResolver: - """Resolver serving fixed bytes and recording every request.""" - - def __init__(self, manifest, status=200): - self._manifest = manifest - self._status = status - self._lock = threading.Lock() - self.requests = [] - - @property - def call_count(self): - with self._lock: - return len(self.requests) - - def resolve(self, request): - with self._lock: - self.requests.append(request) - return HttpResponse(self._status, self._manifest) - - -class HttpResolverTestBase(unittest.TestCase): - """Builds the offline remote-manifest fixture once for all tests.""" - - @classmethod - def setUpClass(cls): - builder = Builder(MANIFEST_DEFINITION) - builder.set_no_embed() - builder.set_remote_url(REMOTE_URL) - output = io.BytesIO() - with open(os.path.join(FIXTURES, "A.jpg"), "rb") as source: - cls.manifest_bytes = builder.sign( - _make_signer(), "image/jpeg", source, output) - cls.asset_bytes = output.getvalue() - - def asset_stream(self): - return io.BytesIO(self.asset_bytes) - - -class TestResolverInvocation(HttpResolverTestBase): - - def test_resolver_invoked_on_remote_read(self): - resolver = CountingResolver(self.manifest_bytes) - with Context.builder().with_resolver(resolver).build() as ctx: - reader = Reader("image/jpeg", self.asset_stream(), context=ctx) - manifest_store = json.loads(reader.json()) - - self.assertEqual(resolver.call_count, 1) - request = resolver.requests[0] - self.assertEqual(request.method, "GET") - self.assertEqual(request.url, REMOTE_URL) - # Request data is copied out of native memory, so it stays usable. - self.assertIsInstance(request.url, str) - self.assertIsInstance(request.method, str) - self.assertIsInstance(request.headers, dict) - self.assertIsInstance(request.body, bytes) - # A manifest fetch is a GET with no body. - self.assertEqual(request.body, b"") - self.assertTrue(manifest_store.get("active_manifest")) - - def test_with_resolver_accepts_callable_and_object(self): - calls = [] - - def resolve_fn(request): - calls.append(request) - return HttpResponse(200, self.manifest_bytes) - - with Context.builder().with_resolver(resolve_fn).build() as ctx: - Reader("image/jpeg", self.asset_stream(), context=ctx) - self.assertEqual(len(calls), 1) - - resolver = CountingResolver(self.manifest_bytes) - with Context.builder().with_resolver(resolver).build() as ctx: - Reader("image/jpeg", self.asset_stream(), context=ctx) - self.assertEqual(resolver.call_count, 1) - - def test_non_callable_resolver_rejected(self): - # Rejected while coercing, before any native resolver is created. - with self.assertRaises(C2paError): - Context.builder().with_resolver(42).build() - - def test_parse_header_lines(self): - # The native side sends an empty string, never NULL, when a request - # carries no headers. - self.assertEqual(_parse_header_lines(""), {}) - self.assertEqual( - _parse_header_lines("accept: */*\nx-token: abc\n"), - {"accept": "*/*", "x-token": "abc"}) - # Repeated names arrive on separate lines; the last one wins. - self.assertEqual( - _parse_header_lines("a: 1\na: 2\n"), {"a": "2"}) - - -class TestResolverErrorPaths(HttpResolverTestBase): - - def test_non_200_statuses(self): - for status in (404, 500, 204, 301): - with self.subTest(status=status): - body = b"boom" if status == 500 else b"" - - def resolve_fn(request, status=status, body=body): - return HttpResponse(status, body) - - with Context.builder().with_resolver( - resolve_fn).build() as ctx: - with self.assertRaises(C2paError) as caught: - Reader("image/jpeg", self.asset_stream(), context=ctx) - # The status code is a stable signal; the wording is not. - self.assertIn(str(status), str(caught.exception)) - - # The process is still healthy afterwards. - resolver = CountingResolver(self.manifest_bytes) - with Context.builder().with_resolver(resolver).build() as ctx: - Reader("image/jpeg", self.asset_stream(), context=ctx) - self.assertEqual(resolver.call_count, 1) - - def test_resolver_exception_is_contained(self): - sentinel = "resolver_sentinel_9f3a" - - def resolve_fn(request): - raise RuntimeError(sentinel) - - with Context.builder().with_resolver(resolve_fn).build() as ctx: - with self.assertRaises(C2paError) as caught: - Reader("image/jpeg", self.asset_stream(), context=ctx) - # Only assert on text this test injected itself. - self.assertIn(sentinel, str(caught.exception)) - - # The error slot is not sticky: a fresh context still works. - resolver = CountingResolver(self.manifest_bytes) - with Context.builder().with_resolver(resolver).build() as ctx: - reader = Reader("image/jpeg", self.asset_stream(), context=ctx) - self.assertTrue(json.loads(reader.json()).get("active_manifest")) - - def test_resolver_bad_return_types(self): - cases = { - "none": lambda request: None, - "no_status_attr": lambda request: object(), - "str_body": lambda request: HttpResponse(200, "not-bytes"), - } - for name, resolve_fn in cases.items(): - with self.subTest(case=name): - with Context.builder().with_resolver( - resolve_fn).build() as ctx: - # Contained as a typed error, never a crash. - with self.assertRaises(C2paError): - Reader("image/jpeg", self.asset_stream(), context=ctx) - - -class TestResolverLifetime(HttpResolverTestBase): - - def test_callback_alive_after_context_close(self): - """The resolver must survive Context.close(). - - The native context is an Arc that Builder clones, so the resolver - can be called after the Python Context is closed. Regression test - for keeping _http_resolver_cb alive in _release(). - """ - resolver = CountingResolver(self.manifest_bytes) - ctx = Context.builder().with_resolver(resolver).build() - builder = Builder(MANIFEST_DEFINITION, context=ctx) - - ctx.close() - del ctx - gc.collect() - - builder.add_ingredient( - {"title": "remote ingredient"}, "image/jpeg", self.asset_stream()) - self.assertEqual(resolver.call_count, 1) - - def test_context_lifecycle_with_resolver(self): - resolver = CountingResolver(self.manifest_bytes) - ctx = Context.builder().with_resolver(resolver).build() - self.assertTrue(ctx.is_valid) - - ctx.close() - ctx.close() # idempotent - self.assertFalse(ctx.is_valid) - with self.assertRaises(C2paError): - ctx._ensure_valid_state() - - with Context.builder().with_resolver(resolver).build() as ctx2: - self.assertTrue(ctx2.is_valid) - - def test_init_attrs_invariant(self): - # _init_attrs must define the attribute, so instances built by - # _wrap_native_handle (which skips __init__) always have it. - ctx = Context() - try: - self.assertIsNone(ctx._http_resolver_cb) - finally: - ctx.close() - - def test_shared_context_threaded(self): - resolver = CountingResolver(self.manifest_bytes) - errors = [] - - def read_once(): - try: - Reader("image/jpeg", self.asset_stream(), context=ctx) - except BaseException as e: # noqa: B036 - report, don't swallow - errors.append(e) - - with Context.builder().with_resolver(resolver).build() as ctx: - threads = [threading.Thread(target=read_once) for _ in range(8)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual(errors, []) - self.assertGreaterEqual(resolver.call_count, 8) - - def test_default_paths_untouched(self): - # No context at all: the built-in resolver path is unchanged. - with open(os.path.join(FIXTURES, "C.jpg"), "rb") as f: - reader = Reader("image/jpeg", f) - self.assertTrue(json.loads(reader.json()).get("active_manifest")) - - # A context with no resolver still takes the c2pa_context_new fast - # path. - with Context() as ctx: - self.assertTrue(ctx.is_valid) - - -class TestResolverSettingsInteraction(HttpResolverTestBase): - - def test_settings_gate_resolver(self): - resolver = CountingResolver(self.manifest_bytes) - ctx = Context.from_dict( - {"verify": {"remote_manifest_fetch": False}}, resolver=resolver) - try: - with self.assertRaises(C2paError): - Reader("image/jpeg", self.asset_stream(), context=ctx) - # Settings gate the resolver: it is never consulted. - self.assertEqual(resolver.call_count, 0) - finally: - ctx.close() - - def test_settings_resolver_and_signer_together(self): - """The full builder path: settings, then resolver, then signer.""" - resolver = CountingResolver(self.manifest_bytes) - ctx = Context.from_dict( - {"verify": {"remote_manifest_fetch": True}}, - signer=_make_signer(), - resolver=resolver) - try: - builder = Builder(MANIFEST_DEFINITION, context=ctx) - builder.add_ingredient( - {"title": "first"}, "image/jpeg", self.asset_stream()) - builder.add_ingredient( - {"title": "second"}, "image/jpeg", self.asset_stream()) - self.assertEqual(resolver.call_count, 2) - - output = io.BytesIO() - with open(os.path.join(FIXTURES, "A.jpg"), "rb") as source: - builder.sign(_make_signer(), "image/jpeg", source, output) - self.assertGreater(len(output.getvalue()), 0) - - # Reading the signed output back uses the embedded manifest, so - # it needs no further fetches. - before = resolver.call_count - reader = Reader( - "image/jpeg", io.BytesIO(output.getvalue()), context=ctx) - self.assertTrue(json.loads(reader.json()).get("active_manifest")) - self.assertEqual(resolver.call_count, before) - finally: - ctx.close() - - -def _current_rss_mb(): - """Resident set size of this process, in MB. - - Deliberately not resource.getrusage(): ru_maxrss is a high-water mark, - so once a peak is reached a later leak stays hidden underneath it. A - leak sentinel needs the current value. - """ - with open("/proc/self/statm") as f: - return int(f.read().split()[1]) * os.sysconf("SC_PAGE_SIZE") / 1048576 - - -def _current_rss_mb_darwin(): - import subprocess - out = subprocess.run( - ["ps", "-o", "rss=", "-p", str(os.getpid())], - capture_output=True, text=True, check=True) - return int(out.stdout.strip()) / 1024 - - -@unittest.skipUnless(sys.platform in ("linux", "darwin"), - "RSS sampling is implemented for Linux and macOS only") -class TestResolverMemory(HttpResolverTestBase): - """Leak sentinels for the response-body ownership contract. - - The native library frees the response body on BOTH return paths, so the - binding must hand the buffer over and never free it afterwards. True - growth is ~0 MB; the threshold only absorbs allocator noise. - - The payload is written into every buffer on purpose. malloc alone does - not commit pages, so an untouched leak would not move RSS and the - sentinel would pass while leaking. Calibrated by injecting the - body_len == 0 leak: 200 iterations of 256 KB grows ~50 MB, well clear - of the threshold. - """ - - ITERATIONS = 200 - CHUNK = 256 * 1024 - THRESHOLD_MB = 20 - - @staticmethod - def _rss_mb(): - if sys.platform == "darwin": - return _current_rss_mb_darwin() - return _current_rss_mb() - - def _assert_rss_stable(self, run_once, label): - # Warm up so first-call allocations are not counted as growth. - for _ in range(5): - run_once() - gc.collect() - before = self._rss_mb() - for _ in range(self.ITERATIONS): - run_once() - gc.collect() - growth = self._rss_mb() - before - would_leak = self.ITERATIONS * self.CHUNK / 1048576 - self.assertLess( - growth, self.THRESHOLD_MB, - f"{label}: RSS grew {growth:.1f} MB " - f"(a full leak would be ~{would_leak:.0f} MB)") - - def test_repeated_cycles_no_leak(self): - """Success path: the native side copies the body then frees it.""" - payload = b"x" * self.CHUNK - - def resolve_fn(request): - # Not a valid manifest, so the read fails; the body is still - # handed to the native side and must still be freed. - return HttpResponse(200, payload) - - def run_once(): - with Context.builder().with_resolver(resolve_fn).build() as ctx: - try: - Reader("image/jpeg", self.asset_stream(), context=ctx) - except C2paError: - pass - - self._assert_rss_stable(run_once, "success path") - - def test_valid_manifest_cycles_no_leak(self): - """Success path with a real manifest, which is fully parsed.""" - resolver = CountingResolver(self.manifest_bytes) - - def run_once(): - with Context.builder().with_resolver(resolver).build() as ctx: - Reader("image/jpeg", self.asset_stream(), context=ctx) - - self._assert_rss_stable(run_once, "valid manifest") - - def test_error_path_body_is_freed(self): - """Error path: a body left behind on a non-zero return is freed. - - Uses a raw callback so the buffer is deliberately assigned before - returning -1, which the binding's own trampoline never does. - """ - chunk = self.CHUNK - malloc = _get_native_malloc() - - def raw_callback(_ctx, request_ptr, response_ptr): - buf = malloc(chunk) - ctypes.memmove(buf, b"x" * chunk, chunk) - response = response_ptr.contents - response.body = ctypes.cast(buf, ctypes.POINTER(ctypes.c_ubyte)) - response.body_len = chunk - _lib.c2pa_error_set_last(b"Other: forced resolver failure") - return -1 - - callback_cb = HttpResolverCallback(raw_callback) - resolver_ptr = _lib.c2pa_http_resolver_create(None, callback_cb) - self.assertTrue(resolver_ptr) - - builder_ptr = _lib.c2pa_context_builder_new() - self.assertEqual( - _lib.c2pa_context_builder_set_http_resolver( - builder_ptr, resolver_ptr), 0) - context_ptr = _lib.c2pa_context_builder_build(builder_ptr) - self.assertTrue(context_ptr) - - wrapped = Context._wrap_native_handle(context_ptr) - # Keep the thunk alive for as long as the context can call it. - wrapped._http_resolver_cb = callback_cb - try: - def run_once(): - try: - Reader("image/jpeg", self.asset_stream(), context=wrapped) - except C2paError: - pass - - self._assert_rss_stable(run_once, "error path") - finally: - wrapped.close() - - def test_empty_body_is_null_not_zero_length(self): - """An empty body must be emitted as NULL, not a zero-length buffer. - - The native side skips its free when body_len == 0, so a non-NULL - pointer with a zero length is never freed and leaks. Rather than - infer this from RSS, drive the trampoline directly and inspect the - response struct it fills in. - """ - from c2pa.c2pa import (C2paHttpRequest, C2paHttpResponse, - _make_http_resolver_trampoline) - - callback_cb = _make_http_resolver_trampoline( - lambda request: HttpResponse(200, b"")) - - request = C2paHttpRequest( - url=b"http://example.test/m.c2pa", method=b"GET", - headers=b"", body=None, body_len=0) - response = C2paHttpResponse(status=0, body=None, body_len=0) - - rc = callback_cb( - None, ctypes.byref(request), ctypes.byref(response)) - - self.assertEqual(rc, 0) - self.assertEqual(response.status, 200) - self.assertEqual(response.body_len, 0) - # The pointer must be NULL, so the native layer has nothing to free. - self.assertFalse(bool(response.body)) - - def test_non_empty_body_is_handed_over(self): - """A non-empty body arrives as a native buffer of the right size.""" - from c2pa.c2pa import (C2paHttpRequest, C2paHttpResponse, - _make_http_resolver_trampoline) - - payload = b"manifest-bytes" - callback_cb = _make_http_resolver_trampoline( - lambda request: HttpResponse(200, payload)) - - request = C2paHttpRequest( - url=b"http://example.test/m.c2pa", method=b"GET", - headers=b"", body=None, body_len=0) - response = C2paHttpResponse(status=0, body=None, body_len=0) - - rc = callback_cb( - None, ctypes.byref(request), ctypes.byref(response)) - try: - self.assertEqual(rc, 0) - self.assertEqual(response.body_len, len(payload)) - self.assertTrue(bool(response.body)) - self.assertEqual( - ctypes.string_at(response.body, response.body_len), payload) - finally: - # Nothing consumed this buffer, so free it here to keep the test - # itself leak-free. Never do this once the SDK has taken it. - if response.body: - _free_native_buffer(response.body) - - -if __name__ == "__main__": - unittest.main() From 769817cd08695fc7fdcdf3725955c015e41569d0 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 13:13:27 -0700 Subject: [PATCH 43/67] fix: Network stuff --- examples/README.md | 37 ------------------------------------- tests/network/README.MD | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 37 deletions(-) create mode 100644 tests/network/README.MD diff --git a/examples/README.md b/examples/README.md index cd8d2f8f..191e88f4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -68,41 +68,6 @@ except Exception as err: sys.exit(err) ``` -## Using a custom HTTP resolver - -A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. - -A resolver is either an object with a `resolve(request)` method or a plain callable. It receives a request object exposing `.url`, `.method`, `.headers` (dict), and `.body` (bytes), and must return a response object exposing `.status` (int) and `.body` (bytes) — any object with those attributes works, there is no type to import: - -```py -class AnHttpResolver: - def resolve(self, request): - # request.url, request.method, request.headers, request.body - return SomeResponse(status=200, body=b"...") - -context = c2pa.Context.builder().with_resolver(MyResolver()).build() -reader = c2pa.Reader("image/jpeg", stream, context=context) -``` - -Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. - -Two things to note before writing one: - -- A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. -- The SDK does not follow redirects by default. Delegating to `urllib.request` in the examples below gives you redirect handling for free. - -Custom resolvers need a live remote manifest to demonstrate against, so their examples live as runnable tests rather than standalone scripts: - -The [`tests/network/test_http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_debug.py) test logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. - -The [`tests/network/test_http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_cache.py) test adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. - -Both use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when it can't reach the network. Run them with: - -```bash -python -m pytest tests/network/ -v -``` - ## Running the examples To run the examples, make sure you have the c2pa-python package installed (`pip install c2pa-python`) and you're in the root directory of the project. We recommend working using virtual environments (venv). Then run the examples as shown below. @@ -135,8 +100,6 @@ In this example, `SignerInfo` creates a `Signer` object that signs the manifest. python examples/sign_info.py ``` -See [Using a custom HTTP resolver](#using-a-custom-http-resolver) above for the debugging and caching resolver examples — they need internet access, so they live under `tests/network/` as runnable (skip-if-offline) tests instead of standalone scripts here. - ## Backend application example [c2pa-python-example](https://github.com/contentauth/c2pa-python-example) is an example of a simple application that accepts an uploaded JPEG image file, attaches a C2PA manifest, and signs it using a certificate. The app uses the CAI Python library and the Flask Python framework to implement a back-end REST endpoint; it does not have an HTML front-end, so you have to use something like curl to access it. This example is a development setup and should not be deployed as-is to a production environment. diff --git a/tests/network/README.MD b/tests/network/README.MD new file mode 100644 index 00000000..eff3396d --- /dev/null +++ b/tests/network/README.MD @@ -0,0 +1,34 @@ +## Using a custom HTTP resolver + +A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. + +A resolver is either an object with a `resolve(request)` method or a plain callable. It receives a request object exposing `.url`, `.method`, `.headers` (dict), and `.body` (bytes), and must return a response object exposing `.status` (int) and `.body` (bytes) — any object with those attributes works, there is no type to import: + +```py +class AnHttpResolver: + def resolve(self, request): + # request.url, request.method, request.headers, request.body + return SomeResponse(status=200, body=b"...") + +context = c2pa.Context.builder().with_resolver(MyResolver()).build() +reader = c2pa.Reader("image/jpeg", stream, context=context) +``` + +Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. + +Two things to note before writing one: + +- A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. +- The SDK does not follow redirects by default. Delegating to `urllib.request` in the examples below gives you redirect handling for free. + +Custom resolvers need a live remote manifest to demonstrate against, so their examples live as runnable tests rather than standalone scripts: + +The [`tests/network/test_http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_debug.py) test logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. + +The [`tests/network/test_http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_cache.py) test adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. + +Both use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when it can't reach the network. Run them with: + +```bash +python -m pytest tests/network/ -v +``` From 0350696b7a7582144811b47bd75ded5e81b2c282 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 14:45:27 -0700 Subject: [PATCH 44/67] fix: Refactor 1 --- src/c2pa/c2pa.py | 142 +++++------ tests/network/README.MD | 34 --- tests/network/README.md | 57 +++++ tests/network/http_resolver.py | 252 +++++++++++++++++++ tests/network/network_test_helpers.py | 46 ++++ tests/network/test_http_resolver_cache.py | 167 ++---------- tests/network/test_http_resolver_contract.py | 108 ++++++++ tests/network/test_http_resolver_debug.py | 86 +------ 8 files changed, 555 insertions(+), 337 deletions(-) delete mode 100644 tests/network/README.MD create mode 100644 tests/network/README.md create mode 100644 tests/network/http_resolver.py create mode 100644 tests/network/network_test_helpers.py create mode 100644 tests/network/test_http_resolver_contract.py diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index d7fc9ddc..a177a11f 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1510,14 +1510,23 @@ class NetworkResource(ManagedResource): Private implementation detail of the custom-resolver feature: not part of the public API. Nests every native mirror/helper the feature needs (ctypes structs, the malloc used for response bodies, header parsing, - resolver normalization, the ctypes trampoline builder, and the request/ - response data shapes) so nothing about this feature is part of the - public surface. Context is the only thing that uses this class. - - A custom resolver never needs to import HttpRequest/HttpResponse: it - receives a request-like object (.url, .method, .headers, .body) and - may return any response-like object (.status, .body) -- duck typing, - not isinstance checks, is what _make_trampoline relies on below. + resolver normalization, and the ctypes trampoline builder) so nothing + about the FFI plumbing is part of the public surface. Context is the + only thing that uses this class. + + This class itself is not re-exported from the top-level c2pa package + or listed in its __all__ -- it's still a real, directly importable + class (`from c2pa.c2pa import NetworkResource`), just excluded from + the curated surface next to Reader/Builder/Context. + + c2pa ships no HttpRequest/HttpResponse/HttpResolver types at all: a + resolver passed to ContextBuilder.with_resolver()/Context(resolver=...) + is never isinstance-checked. Any callable, or any object exposing a + resolve(request) method, is accepted (see _coerce_resolver below); the + request handed to it exposes .url/.method/.headers/.body, and the + value it returns only needs .status (int) and .body (bytes) attributes + -- pure duck typing, both ways. tests/network/http_resolver.py has a + copyable reference implementation of that shape. """ class _Request(ctypes.Structure): @@ -1563,21 +1572,15 @@ class _Handle(ctypes.Structure): ctypes.POINTER(_Request), ctypes.POINTER(_Response)) - class HttpRequest: - """An HTTP request the SDK asks a custom resolver to perform. - - Attributes: - url: Absolute request URL. - method: HTTP method ("GET", "POST", ...). - headers: Request headers as a dict. Names are lowercased by the - native layer; when a header repeats, the last value wins. - body: Request body bytes (b"" when there is none). Timestamp - requests POST a body; manifest fetches send none. - - All data is copied out of native memory, so it stays valid after - the resolver call returns. Not a public type: a resolver reads - attributes off the instance it's handed, it never imports this - class. + class _RequestView: + """Plain Python view of a decoded native HTTP request. + + Internal plumbing only: the trampoline hands one of these to the + resolver's resolve_fn so it has .url/.method/.headers/.body to + read. Not a public type -- callers never construct or import this, + they just read attributes off the instance they're handed. See + tests/network/http_resolver.py for a copyable reference + HttpRequest with the same shape. """ __slots__ = ("url", "method", "headers", "body") @@ -1589,30 +1592,9 @@ def __init__( self.body = body def __repr__(self): - return (f"HttpRequest(method={self.method!r}, " + return (f"_RequestView(method={self.method!r}, " f"url={self.url!r}, body_len={len(self.body)})") - class HttpResponse: - """The answer a custom resolver returns to the SDK. - - Attributes: - status: HTTP status code. Remote manifest fetches only accept - 200; any other code surfaces as a typed C2paError. - body: Response body bytes. - - Not a public type: a resolver may return any object exposing these - two attributes, it never needs to import this class. - """ - __slots__ = ("status", "body") - - def __init__(self, status: int, body: bytes = b""): - self.status = status - self.body = body - - def __repr__(self): - return (f"HttpResponse(status={self.status}, " - f"body_len={len(self.body or b'')})") - _native_malloc = None @staticmethod @@ -1674,15 +1656,18 @@ def _coerce_resolver(resolver): return resolve_fn if callable(resolver): return resolver - raise C2paError( - "HTTP resolver must be callable or provide a resolve() method") + raise TypeError( + "HTTP resolver must be callable or provide a resolve() method, " + f"got {type(resolver).__name__}") @classmethod def _make_trampoline(cls, resolve_fn): """Wrap a Python resolve function into a native C callback. Args: - resolve_fn: Callable[[HttpRequest], HttpResponse]. + resolve_fn: Callable taking a duck-typed request (.url/ + .method/.headers/.body) and returning a duck-typed + response (.status/.body) -- see cls._RequestView. Returns: The _Callback object. The caller MUST keep a reference to it @@ -1704,7 +1689,7 @@ def _trampoline(_ctx, request_ptr, response_ptr): body = (ctypes.string_at(req.body, req.body_len) if (req.body and req.body_len) else b"") - result = resolve_fn(cls.HttpRequest( + result = resolve_fn(cls._RequestView( url=url, method=method, headers=cls._parse_header_lines(raw_headers), @@ -1713,7 +1698,7 @@ def _trampoline(_ctx, request_ptr, response_ptr): payload = result.body or b"" if not isinstance(payload, (bytes, bytearray)): raise TypeError( - "HttpResponse.body must be bytes, got " + "resolver response .body must be bytes, got " f"{type(payload).__name__}") response = response_ptr.contents @@ -1957,25 +1942,35 @@ def with_resolver( """Attach a custom HTTP resolver to the Context being built. The resolver handles every HTTP request the SDK makes through this - Context: remote manifest fetches, OCSP requests, RFC 3161 timestamp - requests, and CAWG did:web resolution. Reader and Builder instances - created without a Context keep using the built-in resolver. - - Can be called multiple times; the last resolver wins. + Context: remote manifest fetches, OCSP requests, RFC 3161 + timestamp requests, and CAWG did:web resolution. Reader and + Builder instances created without a Context keep using the + built-in resolver. Can be called multiple times; the last + resolver wins. + + c2pa ships no resolver type to subclass or construct: resolver is + never isinstance-checked. Any callable, or any object with a + resolve(request) method, works -- the request it receives exposes + .url (str), .method (str), .headers (dict), and .body (bytes); + the response it returns only needs .status (int) and .body + (bytes) attributes. See tests/network/http_resolver.py for a + copyable reference implementation of that shape. Validated + immediately: an invalid resolver raises TypeError here, not later + at build(). Args: - resolver: An object with a resolve(request) method, or a - callable with the same signature. It receives a request - object exposing .url (str), .method (str), .headers - (dict), and .body (bytes), and must return a response - object exposing .status (int) and .body (bytes) -- any - object with those attributes works, there is no type to - import. Raising instead marks the request as a hard - failure, which surfaces as a typed C2paError. + resolver: A callable, or an object with a resolve(request) + method, matching the duck-typed contract above. Raising + from the resolver marks the request as a hard failure, + which surfaces as a typed C2paError. Returns: self, for method chaining. + Raises: + TypeError: resolver is neither callable nor has a resolve() + method. + Notes: - Only status 200 is accepted for a remote manifest fetch; any other status surfaces as a typed C2paError. @@ -1998,6 +1993,7 @@ def with_resolver( - Do not call c2pa APIs from inside the resolver: reentering the FFI while a call is in flight is undefined. """ + NetworkResource._coerce_resolver(resolver) self._resolver = resolver return self @@ -2055,6 +2051,8 @@ def __init__( Raises: C2paError: If creation fails + TypeError: resolver is neither callable nor has a resolve() + method. """ super().__init__() self._init_attrs() @@ -2112,23 +2110,7 @@ def _init_attrs(self): self._http_resolver_cb = None def _release(self): - """Release Context-specific resources. - - _http_resolver_cb is deliberately NOT cleared here. The native - context is an Arc, and c2pa_reader_from_context / - c2pa_builder_from_context clone it, so the native context (and the - resolver holding a raw pointer into the ctypes thunk) can outlive - this object's ACTIVE state. Reader and Builder keep a reference to - the Python Context for their whole lifetime, and that reference - chain is what keeps the thunk alive as long as any native clone can - still call it. Clearing it here frees the thunk while it is still - reachable from native code, which is undefined behavior: an observed - symptom is a garbage response reported as "invalid status code". - - The asymmetry with _signer_callback_cb below is intentional. That - one has the same latent hazard, but it predates this code and - changing it is out of scope here. - """ + """Release Context-specific resources.""" self._signer_callback_cb = None @classmethod diff --git a/tests/network/README.MD b/tests/network/README.MD deleted file mode 100644 index eff3396d..00000000 --- a/tests/network/README.MD +++ /dev/null @@ -1,34 +0,0 @@ -## Using a custom HTTP resolver - -A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. - -A resolver is either an object with a `resolve(request)` method or a plain callable. It receives a request object exposing `.url`, `.method`, `.headers` (dict), and `.body` (bytes), and must return a response object exposing `.status` (int) and `.body` (bytes) — any object with those attributes works, there is no type to import: - -```py -class AnHttpResolver: - def resolve(self, request): - # request.url, request.method, request.headers, request.body - return SomeResponse(status=200, body=b"...") - -context = c2pa.Context.builder().with_resolver(MyResolver()).build() -reader = c2pa.Reader("image/jpeg", stream, context=context) -``` - -Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. - -Two things to note before writing one: - -- A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. -- The SDK does not follow redirects by default. Delegating to `urllib.request` in the examples below gives you redirect handling for free. - -Custom resolvers need a live remote manifest to demonstrate against, so their examples live as runnable tests rather than standalone scripts: - -The [`tests/network/test_http_resolver_debug.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_debug.py) test logs the method and URL of each request and the status of each response, delegating the transfer to `urllib`. - -The [`tests/network/test_http_resolver_cache.py`](https://github.com/contentauth/c2pa-python/blob/main/tests/network/test_http_resolver_cache.py) test adds an LRU cache with a TTL (defaults: 100 items, 120 seconds) and retries throttled requests. Only GET requests answered with 200 are cached. - -Both use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when it can't reach the network. Run them with: - -```bash -python -m pytest tests/network/ -v -``` diff --git a/tests/network/README.md b/tests/network/README.md new file mode 100644 index 00000000..5c7921c6 --- /dev/null +++ b/tests/network/README.md @@ -0,0 +1,57 @@ +## Using a custom HTTP resolver + +A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. + +c2pa ships no resolver types at all — no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `with_resolver()` never does an `isinstance` check: it accepts any callable, or any object with a `resolve(request)` method. The request it receives exposes `.url`, `.method`, `.headers` (dict), and `.body` (bytes); the response it returns only needs `.status` (int) and `.body` (bytes) attributes. + +The minimal form needs no imports at all: + +```py +def my_resolver(request): + # request.url, request.method, request.headers, request.body + return SomeResponse(status=200, body=b"...") + +context = c2pa.Context.builder().with_resolver(my_resolver).build() +reader = c2pa.Reader("image/jpeg", stream, context=context) +``` + +[`http_resolver.py`](./http_resolver.py) in this directory is a reference implementation of that shape — copy it into your own project or adapt it. It has `HttpRequest`, `HttpResponse`, an optional `HttpResolver` base class (subclassing it is not required — it just gets you a documented, type-checkable contract instead of duck typing), plus two example resolvers this test suite exercises: `CachingHttpResolver` (LRU cache with a TTL and retry/backoff for throttled requests) and `DebugHttpResolver` (logs every request/response, delegating the transfer to `urllib`). Neither example imports `c2pa` itself — only the `test_http_resolver_*.py` files do, to exercise them against real `Context`/`Reader`/`Builder` instances. + +Whichever form you use, it's validated immediately: passing something with neither shape raises `TypeError` from `with_resolver()` itself, not later at `.build()`. Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. + +Two things to note before writing one: + +- A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. +- The SDK does not follow redirects by default. Delegating to `urllib.request` as `http_resolver.py`'s examples do gives you redirect handling for free. + +### How a custom resolver works under the hood + +1. **The wiring.** `ContextBuilder.with_resolver(resolver)` stores the resolver, validating it eagerly. `Context.__init__` (whether reached via the builder or `Context(resolver=...)` directly) wraps it in `NetworkResource` — a private class in `c2pa.py` that owns the native `C2paHttpResolver` handle via `c2pa_http_resolver_create`, following the same `ManagedResource` lifecycle as `Reader`/`Builder`/`Signer`. + +2. **The trampoline.** Your resolver is wrapped into a ctypes `CFUNCTYPE` callback (`NetworkResource._make_trampoline`). The native side calls this callback for every HTTP request the SDK makes through that `Context` — remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG did:web resolution. The callback decodes the native `C2paHttpRequest` struct into plain Python `str`/`bytes`, calls your `resolve()`, and writes the result back into the native `C2paHttpResponse` struct. + +3. **Memory and ownership handling** — the part implementers most often get wrong: + - A non-empty response body must be allocated with the *same C runtime malloc* the native library's `free()` will use (`ucrtbase` before `msvcrt` on Windows, the process's own libc elsewhere). Mismatched allocators cause heap corruption, not a leak. You don't need to worry about this yourself — the trampoline does the allocation and copy for you from the `bytes` your resolver returns. + - Ownership transfers to native code once that buffer is written into the response struct: native frees it on *both* the success path (after copying) and the error path. Nothing on the Python side ever frees it. + - The zero-length trap: an empty body must leave `body`/`body_len` as `NULL`/`0` *together* — a non-NULL pointer with `body_len == 0` is never freed by native code and leaks. The trampoline handles this for you as long as your resolver returns `b""` (not some non-empty placeholder) for an empty body. + - The callback thunk is pinned on the `Context` and deliberately *not* cleared when the `Context` closes: native `Reader`/`Builder` instances clone the underlying Arc, so your resolver can still be invoked by a native clone after `Context.close()`, for as long as that `Reader`/`Builder` is alive. Your resolver must stay valid (and thread-safe — it may be called from SDK worker threads) for that whole lifetime. + - Exceptions cannot unwind across the ctypes/native boundary: the trampoline catches every exception your resolver raises, reports it to the native error slot, and turns it into a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch — never from inside your `resolve()` itself. + +### The reference tests + +The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) test exercises `DebugHttpResolver`: logs the method and URL of each request and the status of each response. + +The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) test exercises `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. + +The [`test_http_resolver_contract.py`](./test_http_resolver_contract.py) test needs no network: it pins the eager-validation behavior above and that c2pa ships no resolver types at all (`HttpRequest`/`HttpResponse`/`HttpResolver` only exist in `http_resolver.py`, not in `c2pa.c2pa`), while `NetworkResource` — the FFI binding itself — stays importable from `c2pa.c2pa`, just not re-exported from the top-level `c2pa` package. + +The debug and cache tests use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when the resolver itself observes a transport failure. Run them with: + +```bash +python ./tests/network/test_http_resolver_contract.py # offline-safe +python ./tests/network/test_http_resolver_debug.py # needs network +python ./tests/network/test_http_resolver_cache.py # needs network + +# Or the whole directory via unittest discovery: +python -m unittest discover -s tests/network -v +``` diff --git a/tests/network/http_resolver.py b/tests/network/http_resolver.py new file mode 100644 index 00000000..88fe8008 --- /dev/null +++ b/tests/network/http_resolver.py @@ -0,0 +1,252 @@ +# Copyright 2025 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, +# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) +# or the MIT license (http://opensource.org/licenses/MIT), +# at your option. + +# Unless required by applicable law or agreed to in writing, +# this software is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or +# implied. See the LICENSE-MIT and LICENSE-APACHE files for the +# specific language governing permissions and limitations under +# each license. + +"""Example implementations of a custom HTTP resolver for c2pa. + +c2pa ships no HttpRequest/HttpResponse/HttpResolver types: +the resolver passed to ContextBuilder.with_resolver()/Context(resolver=...) +is never isinstance-checked, so any of these classes is a starting point to copy +and adapt, not a required dependency. + +This module has no dependency on c2pa itself: it only needs to satisfy +the attribute shape the SDK expects. +""" + +import collections +import threading +import time +import urllib.error +import urllib.request +from abc import ABC, abstractmethod + + +class HttpRequest: + """An HTTP request the SDK asks a custom resolver to perform. + + Attributes: + url: Absolute request URL. + method: HTTP method ("GET", "POST", ...). + headers: Request headers as a dict. Names are lowercased by the + native layer; when a header repeats, the last value wins. + body: Request body bytes (b"" when there is none). Timestamp + requests POST a body; manifest fetches send none. + + Reference shape only: + c2pa hands your resolve() a duck-typed object with these same four attributes, + not necessarily an instance of this exact class. + Define your own compatible type, or just use this one. + """ + __slots__ = ("url", "method", "headers", "body") + + def __init__(self, url: str, method: str, headers: dict, body: bytes): + self.url = url + self.method = method + self.headers = headers + self.body = body + + def __repr__(self): + return (f"HttpRequest(method={self.method!r}, url={self.url!r}, " + f"body_len={len(self.body)})") + + +class HttpResponse: + """The answer a custom resolver returns to the SDK. + + Attributes: + status: HTTP status code. Remote manifest fetches only accept 200; + any other code surfaces as a typed C2paError. + body: Response body bytes. + + Reference shape only: + c2pa accepts any object exposing these two attributes, + not necessarily an instance of this exact class. + """ + __slots__ = ("status", "body") + + def __init__(self, status: int, body: bytes = b""): + self.status = status + self.body = body + + def __repr__(self): + return (f"HttpResponse(status={self.status}, " + f"body_len={len(self.body or b'')})") + + +class HttpResolver(ABC): + """Optional base class for custom HTTP resolvers. + + Attach an instance via ContextBuilder.with_resolver() or the + Context(resolver=...) constructor kwarg. + """ + + @abstractmethod + def resolve(self, request: HttpRequest) -> HttpResponse: + """Perform the request. + + Raising instead marks the request as a hard failure, which + surfaces as a typed C2paError. + """ + raise NotImplementedError + + +class TtlLruCache: + """A small LRU cache whose entries also expire after a TTL.""" + + def __init__(self, max_items=100, ttl_seconds=120.0): + self._max_items = int(max_items) + self._ttl = float(ttl_seconds) + self._entries = collections.OrderedDict() # key -> (expiry, value) + self._lock = threading.Lock() + self.hits = 0 + self.misses = 0 + + def get(self, key): + with self._lock: + entry = self._entries.get(key) + # time.monotonic, not time.time: immune to wall-clock jumps. + if entry is not None and entry[0] > time.monotonic(): + self._entries.move_to_end(key) + self.hits += 1 + return entry[1] + if entry is not None: + del self._entries[key] + self.misses += 1 + return None + + def put(self, key, value): + with self._lock: + if key in self._entries: + self._entries.move_to_end(key) + self._entries[key] = (time.monotonic() + self._ttl, value) + while len(self._entries) > self._max_items: + self._entries.popitem(last=False) + + +class CachingHttpResolver(HttpResolver): + """An HTTP resolver with a response cache and bounded retries. + + Caching policy: + - Only read GET requests answered with 200 are cached. + - POSTs (timestamp requests) and error responses are not. + + Retry policy: 429 and 503 are retried up to max_retries times, + honoring a capped Retry-After when the header is present, + and otherwise backing off exponentially. + Any other status is final and is passed through to the SDK. + Transport errors raise, which marks a hard failure. + """ + + def __init__(self, cache=None, timeout=10.0, max_retries=3, + backoff_seconds=0.5, max_retry_after=10.0): + self.cache = cache if cache is not None else TtlLruCache() + self._timeout = timeout + self._max_retries = int(max_retries) + self._backoff = float(backoff_seconds) + self._max_retry_after = float(max_retry_after) + self.transport_errors = [] + + def resolve(self, request): + cacheable = request.method.upper() == "GET" + if cacheable: + cached = self.cache.get(request.url) + if cached is not None: + return HttpResponse(cached[0], cached[1]) + + status, body = self._fetch_with_retries(request) + if cacheable and status == 200: + self.cache.put(request.url, (status, body)) + return HttpResponse(status, body) + + def _fetch_with_retries(self, request): + data = request.body or None + for attempt in range(self._max_retries + 1): + req = urllib.request.Request( + request.url, + data=data, + method=request.method, + headers=request.headers) + try: + with urllib.request.urlopen( + req, timeout=self._timeout) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + retryable = e.code in (429, 503) + if not retryable or attempt == self._max_retries: + # Final failure: pass the status back so the SDK can + # report it. + return e.code, e.read() + delay = self._retry_delay(e, attempt) + time.sleep(delay) + except urllib.error.URLError as e: + # DNS failure, connection refused, timeout: recorded so + # network_test_helpers.skip_if_offline can tell this apart + # from a real bug. + self.transport_errors.append(e) + raise + raise RuntimeError("unreachable") + + def _retry_delay(self, error, attempt): + retry_after = error.headers.get("Retry-After") + if retry_after: + try: + return min(float(retry_after), self._max_retry_after) + except ValueError: + pass + return self._backoff * (2 ** attempt) + + +class AlwaysFailResolver(HttpResolver): + """Answers every request with a fixed status. Needs no network.""" + + def __init__(self, status): + self._status = status + + def resolve(self, request): + return HttpResponse(self._status, b"") + + +class DebugHttpResolver(HttpResolver): + """Logs every SDK HTTP request and response status. + The actual transfer is delegated to urllib. + """ + + def __init__(self, timeout=10.0): + self._timeout = timeout + self.requests = [] + self.transport_errors = [] + + def resolve(self, request): + self.requests.append((request.method, request.url)) + + # Timestamp requests POST a body; manifest fetches send none. + data = request.body or None + req = urllib.request.Request( + request.url, + data=data, + method=request.method, + headers=request.headers) + + try: + with urllib.request.urlopen(req, timeout=self._timeout) as resp: + return HttpResponse(resp.status, resp.read()) + except urllib.error.HTTPError as e: + # A 4xx/5xx is still a response! Pass the status through and + # let the SDK turn it into its own typed error: a remote + # manifest fetch only accepts 200. + return HttpResponse(e.code, e.read()) + except urllib.error.URLError as e: + # DNS failure, connection refused, timeout: recorded so + # network_test_helpers.skip_if_offline can tell this apart + # from a real bug. + self.transport_errors.append(e) + raise diff --git a/tests/network/network_test_helpers.py b/tests/network/network_test_helpers.py new file mode 100644 index 00000000..25298f9f --- /dev/null +++ b/tests/network/network_test_helpers.py @@ -0,0 +1,46 @@ +# Copyright 2025 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, +# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) +# or the MIT license (http://opensource.org/licenses/MIT), +# at your option. + +# Unless required by applicable law or agreed to in writing, +# this software is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or +# implied. See the LICENSE-MIT and LICENSE-APACHE files for the +# specific language governing permissions and limitations under +# each license. + +"""Shared helpers for the tests/network resolver reference tests. + +These tests exercise a custom HTTP resolver against the real network +(tests/fixtures/cloud.jpg only has a remote manifest, no embedded one). +""" + +import os +import sys + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__)))) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + +FIXTURES = os.path.join(_REPO_ROOT, "tests", "fixtures") + + +def skip_if_offline(testcase, resolver, exc): + """Skip the test iff the resolver itself observed a transport failure. + + resolver must record urllib.error.URLError instances it catches (DNS + failure, connection refused, timeout) in a `transport_errors` list + before re-raising. Skipping only on an *observed* transport error -- + rather than sniffing exc's message for substrings like "fetch" or + "resolver" -- means a real bug in the resolver or the SDK still fails + the test instead of being silently skipped: a typed error mentioning + "resolver" (the trampoline's own failure-message prefix) is exactly + the kind of bug this distinction is meant to catch. + """ + if resolver.transport_errors: + testcase.skipTest( + "needs internet access to fetch the remote manifest for " + f"tests/fixtures/cloud.jpg: {resolver.transport_errors[-1]}") + raise exc diff --git a/tests/network/test_http_resolver_cache.py b/tests/network/test_http_resolver_cache.py index 2ad9f7c6..134deaeb 100644 --- a/tests/network/test_http_resolver_cache.py +++ b/tests/network/test_http_resolver_cache.py @@ -11,163 +11,37 @@ # specific language governing permissions and limitations under # each license. -"""A custom HTTP resolver that caches responses and retries throttled -requests, exercised against the real network. +"""Tests for CachingHttpResolver (tests/network/http_resolver.py), +exercised against the real network. -Ported from the former examples/http_resolver_cache.py: still a runnable -demonstration of the resolver pattern, now also verified in CI. Needs -internet access to fetch the remote manifest for tests/fixtures/cloud.jpg; -tests skip (not fail) when that fetch can't reach the network. +Needs internet access to fetch the remote manifest for +tests/fixtures/cloud.jpg; tests skip (not fail) when the resolver itself +observes a transport failure. """ -import collections import os import sys import tempfile -import threading -import time import unittest -import urllib.error -import urllib.request -from typing import NamedTuple -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) - -import c2pa # noqa: E402 - -FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, "fixtures") - - -class HttpResponse(NamedTuple): - """Duck-typed stand-in for c2pa's internal resolver response shape. - - The resolver contract only requires .status (int) and .body (bytes); - there is no public type to import, any object with those attributes - works. - """ - status: int - body: bytes = b"" - - -def _skip_if_offline(testcase, exc): - """Skip the test if exc looks like a network-reachability failure. - - Re-raises anything else, so a real bug in the resolver or the SDK - still fails the test instead of being silently skipped. - """ - message = str(exc) - if "fetch" in message or "resolver" in message: - testcase.skipTest( - "needs internet access to fetch the remote manifest for " - f"tests/fixtures/cloud.jpg: {exc}") - raise exc - - -class TtlLruCache: - """A small LRU cache whose entries also expire after a TTL.""" - - def __init__(self, max_items=100, ttl_seconds=120.0): - self._max_items = int(max_items) - self._ttl = float(ttl_seconds) - self._entries = collections.OrderedDict() # key -> (expiry, value) - self._lock = threading.Lock() - self.hits = 0 - self.misses = 0 - - def get(self, key): - with self._lock: - entry = self._entries.get(key) - # time.monotonic, not time.time: immune to wall-clock jumps. - if entry is not None and entry[0] > time.monotonic(): - self._entries.move_to_end(key) - self.hits += 1 - return entry[1] - if entry is not None: - del self._entries[key] - self.misses += 1 - return None - - def put(self, key, value): - with self._lock: - if key in self._entries: - self._entries.move_to_end(key) - self._entries[key] = (time.monotonic() + self._ttl, value) - while len(self._entries) > self._max_items: - self._entries.popitem(last=False) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from network_test_helpers import FIXTURES, skip_if_offline # noqa: E402 +from http_resolver import AlwaysFailResolver, CachingHttpResolver # noqa: E402,E501 -class CachingHttpResolver: - """An HTTP resolver with a response cache and bounded retries. - - Caching policy: only read GET requests answered with 200 are cached. - POSTs (timestamp requests) and error responses are not. - - Retry policy: 429 and 503 are retried up to max_retries times, - honoring a capped Retry-After when the header is present, - and otherwise backing off exponentially. - Any other status is final and is passed through to the SDK. - Transport errors raise, which marks a hard failure. - """ - - def __init__(self, cache=None, timeout=10.0, max_retries=3, - backoff_seconds=0.5, max_retry_after=10.0): - self.cache = cache if cache is not None else TtlLruCache() - self._timeout = timeout - self._max_retries = int(max_retries) - self._backoff = float(backoff_seconds) - self._max_retry_after = float(max_retry_after) - - def resolve(self, request): - cacheable = request.method.upper() == "GET" - if cacheable: - cached = self.cache.get(request.url) - if cached is not None: - return HttpResponse(cached[0], cached[1]) - - status, body = self._fetch_with_retries(request) - if cacheable and status == 200: - self.cache.put(request.url, (status, body)) - return HttpResponse(status, body) - - def _fetch_with_retries(self, request): - data = request.body or None - for attempt in range(self._max_retries + 1): - req = urllib.request.Request( - request.url, - data=data, - method=request.method, - headers=request.headers) - try: - with urllib.request.urlopen( - req, timeout=self._timeout) as resp: - return resp.status, resp.read() - except urllib.error.HTTPError as e: - retryable = e.code in (429, 503) - if not retryable or attempt == self._max_retries: - # Final failure: pass the status back so the SDK can - # report it. - return e.code, e.read() - delay = self._retry_delay(e, attempt) - time.sleep(delay) - raise RuntimeError("unreachable") - - def _retry_delay(self, error, attempt): - retry_after = error.headers.get("Retry-After") - if retry_after: - try: - return min(float(retry_after), self._max_retry_after) - except ValueError: - pass - return self._backoff * (2 ** attempt) +import c2pa # noqa: E402 class TestHttpResolverCache(unittest.TestCase): def test_read_with_cache(self): - """Reading the same remote-manifest asset twice hits the cache.""" + """Reading the same remote-manifest asset twice hits the cache. + + Assumption this exact-count assert depends on: one remote-manifest + GET per read, and nothing else on this path (no OCSP, no did:web). + If the SDK ever adds another GET here, this count shifts -- that's + expected and worth updating, not a false alarm. + """ resolver = CachingHttpResolver() try: with c2pa.Context.builder().with_resolver( @@ -179,7 +53,7 @@ def test_read_with_cache(self): "image/jpeg", f, context=context) as reader: reader.get_validation_state() except c2pa.C2paError as e: - _skip_if_offline(self, e) + skip_if_offline(self, resolver, e) self.assertEqual(resolver.cache.hits, 1) self.assertEqual(resolver.cache.misses, 1) @@ -257,7 +131,7 @@ def test_sign_with_cache(self): "image/jpeg", source, dest) self.assertTrue(os.path.exists(output_path)) except c2pa.C2paError as e: - _skip_if_offline(self, e) + skip_if_offline(self, resolver, e) finally: context.close() @@ -269,11 +143,8 @@ def test_failing_resolver_is_a_clean_error(self): Needs no network: the resolver answers without calling out. """ - def always_500(request): - return HttpResponse(500, b"") - with c2pa.Context.builder().with_resolver( - always_500).build() as context: + AlwaysFailResolver(500)).build() as context: with self.assertRaises(c2pa.C2paError): with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: c2pa.Reader("image/jpeg", f, context=context) diff --git a/tests/network/test_http_resolver_contract.py b/tests/network/test_http_resolver_contract.py new file mode 100644 index 00000000..82c8a8f2 --- /dev/null +++ b/tests/network/test_http_resolver_contract.py @@ -0,0 +1,108 @@ +# Copyright 2025 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, +# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) +# or the MIT license (http://opensource.org/licenses/MIT), +# at your option. + +# Unless required by applicable law or agreed to in writing, +# this software is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or +# implied. See the LICENSE-MIT and LICENSE-APACHE files for the +# specific language governing permissions and limitations under +# each license. + +"""Offline-safe contract tests for the custom HTTP resolver feature. + +No network needed, unlike test_http_resolver_debug.py and +test_http_resolver_cache.py: these pin the resolver validation behavior +(eager TypeError, not deferred to Context.build()), dual bare-callable/ +HttpResolver support, and that c2pa ships no resolver types at all -- +HttpRequest/HttpResponse/HttpResolver here come from the local reference +implementation (tests/network/http_resolver.py), not from c2pa itself. +NetworkResource is the one piece of this feature that does live in c2pa +(it's the FFI binding), so its "importable but not re-exported" property +is pinned here too. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from http_resolver import HttpResolver, HttpResponse # noqa: E402 + +import c2pa # noqa: E402 +import c2pa.c2pa # noqa: E402 +from c2pa.c2pa import NetworkResource # noqa: E402 + + +class TestResolverValidation(unittest.TestCase): + + def test_with_resolver_rejects_non_callable_eagerly(self): + # Raised by with_resolver() itself, not deferred to build(). + with self.assertRaises(TypeError): + c2pa.Context.builder().with_resolver(42) + + def test_with_resolver_rejects_object_without_resolve_eagerly(self): + with self.assertRaises(TypeError): + c2pa.Context.builder().with_resolver(object()) + + def test_with_resolver_accepts_bare_callable(self): + # Bare callables/duck-typed resolvers are fully supported -- + # c2pa never requires an HttpResolver subclass, or any particular + # class at all. + builder = c2pa.Context.builder().with_resolver( + lambda request: HttpResponse(200, b"")) + self.assertIsInstance(builder, c2pa.ContextBuilder) + + def test_with_resolver_accepts_resolve_method_object(self): + class DuckTypedResolver: + def resolve(self, request): + return HttpResponse(200, b"") + + builder = c2pa.Context.builder().with_resolver(DuckTypedResolver()) + self.assertIsInstance(builder, c2pa.ContextBuilder) + + def test_with_resolver_accepts_http_resolver_subclass(self): + # HttpResolver is the reference implementation's optional base + # class (tests/network/http_resolver.py) -- not required, but + # accepted like anything else with a resolve() method. + class MyResolver(HttpResolver): + def resolve(self, request): + return HttpResponse(200, b"") + + builder = c2pa.Context.builder().with_resolver(MyResolver()) + self.assertIsInstance(builder, c2pa.ContextBuilder) + + def test_context_constructor_rejects_invalid_resolver_eagerly(self): + # Covers the non-builder path: Context(resolver=...) directly. + with self.assertRaises(TypeError): + c2pa.Context(resolver="nope") + + +class TestNoShippedResolverTypes(unittest.TestCase): + """c2pa ships no HttpRequest/HttpResponse/HttpResolver at all: they + aren't reachable from c2pa.c2pa, let alone re-exported from the + top-level c2pa package.""" + + def test_resolver_contract_types_not_in_c2pa_c2pa(self): + for name in ("HttpRequest", "HttpResponse", "HttpResolver"): + self.assertFalse( + hasattr(c2pa.c2pa, name), + f"c2pa.c2pa.{name} should not exist -- the resolver " + "contract is fully duck-typed, with no shipped type. See " + "tests/network/http_resolver.py for a reference " + "implementation.") + + def test_network_resource_importable_but_not_reexported(self): + # NetworkResource is the FFI binding, unlike the resolver contract + # types above -- it stays in c2pa.c2pa, just excluded from + # c2pa.__all__ next to Reader/Builder/Context. + self.assertNotIn("NetworkResource", c2pa.__all__) + self.assertFalse(hasattr(c2pa, "NetworkResource")) + self.assertTrue(NetworkResource) # importable from c2pa.c2pa + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/network/test_http_resolver_debug.py b/tests/network/test_http_resolver_debug.py index adc8dff6..ea2897bc 100644 --- a/tests/network/test_http_resolver_debug.py +++ b/tests/network/test_http_resolver_debug.py @@ -11,14 +11,13 @@ # specific language governing permissions and limitations under # each license. -"""A custom HTTP resolver that logs every request/response, exercised +"""Tests for DebugHttpResolver (tests/network/http_resolver.py), exercised against the real network. -Ported from the former examples/http_resolver_debug.py: still a runnable -demonstration of intercepting every HTTP request the SDK makes through a -Context, now also verified in CI. Needs internet access to fetch the -remote manifest for tests/fixtures/cloud.jpg; tests skip (not fail) when -that fetch can't reach the network. +Demonstrates intercepting every HTTP request the SDK makes through a +Context. Needs internet access to fetch the remote manifest for +tests/fixtures/cloud.jpg; tests skip (not fail) when the resolver itself +observes a transport failure. """ import json @@ -26,76 +25,13 @@ import sys import tempfile import unittest -import urllib.error -import urllib.request -from typing import NamedTuple -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import c2pa # noqa: E402 - -FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, "fixtures") - - -class HttpResponse(NamedTuple): - """Duck-typed stand-in for c2pa's internal resolver response shape. - - The resolver contract only requires .status (int) and .body (bytes); - there is no public type to import, any object with those attributes - works. - """ - status: int - body: bytes = b"" - - -def _skip_if_offline(testcase, exc): - """Skip the test if exc looks like a network-reachability failure. - - Re-raises anything else, so a real bug in the resolver or the SDK - still fails the test instead of being silently skipped. - """ - message = str(exc) - if "fetch" in message or "resolver" in message: - testcase.skipTest( - "needs internet access to fetch the remote manifest for " - f"tests/fixtures/cloud.jpg: {exc}") - raise exc +from network_test_helpers import FIXTURES, skip_if_offline # noqa: E402 +from http_resolver import DebugHttpResolver # noqa: E402 - -class DebugHttpResolver: - """Logs every SDK HTTP request and response status. - The actual transfer is delegated to urllib. - """ - - def __init__(self, timeout=10.0): - self._timeout = timeout - self.requests = [] - - def resolve(self, request): - self.requests.append((request.method, request.url)) - - # Timestamp requests POST a body; manifest fetches send none. - data = request.body or None - req = urllib.request.Request( - request.url, - data=data, - method=request.method, - headers=request.headers) - - try: - with urllib.request.urlopen(req, timeout=self._timeout) as resp: - return HttpResponse(resp.status, resp.read()) - except urllib.error.HTTPError as e: - # A 4xx/5xx is still a response! Pass the status through and - # let the SDK turn it into its own typed error: a remote - # manifest fetch only accepts 200. - return HttpResponse(e.code, e.read()) - # urllib.error.URLError (DNS failure, connection refused, timeout) - # is deliberately not caught: raising marks the request as a hard - # resolver failure, which surfaces as a typed C2paError. +import c2pa # noqa: E402 class TestHttpResolverDebug(unittest.TestCase): @@ -114,7 +50,7 @@ def test_read_with_resolver(self): self.assertFalse(reader.is_embedded()) self.assertTrue(reader.get_remote_url()) except c2pa.C2paError as e: - _skip_if_offline(self, e) + skip_if_offline(self, resolver, e) self.assertTrue( any(method == "GET" for method, _ in resolver.requests)) @@ -197,7 +133,7 @@ def test_sign_with_resolver(self): self.assertEqual( len(resolver.requests), requests_before_reread) except c2pa.C2paError as e: - _skip_if_offline(self, e) + skip_if_offline(self, resolver, e) finally: context.close() From 92024e3c728145ce450c2e4c41b7b764be610053 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 14:51:51 -0700 Subject: [PATCH 45/67] fix: Refactor 2 --- src/c2pa/c2pa.py | 506 +++++++++---------- tests/network/README.md | 8 +- tests/network/test_http_resolver_contract.py | 12 - 3 files changed, 253 insertions(+), 273 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index a177a11f..706b7774 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -809,6 +809,55 @@ class C2paContext(ctypes.Structure): """Opaque structure for context.""" _fields_ = [] # Empty as it's opaque in the C API + +class C2paHttpRequest(ctypes.Structure): + """Matches the native C2paHttpRequest going through the C FFI. + + Read-only view: + Every pointer borrows native memory that is only valid + for the duration of the resolver callback. + Copy anything you want to keep for later on. + + The string fields are c_char_p so ctypes converts them + to bytes on read. + `body` stays a c_ubyte pointer because it is binary, + not NUL-terminated. + """ + _fields_ = [ + ("url", ctypes.c_char_p), + ("method", ctypes.c_char_p), + ("headers", ctypes.c_char_p), + ("body", ctypes.POINTER(ctypes.c_ubyte)), + ("body_len", ctypes.c_size_t), + ] + + +class C2paHttpResponse(ctypes.Structure): + """Mirror of the native C2paHttpResponse (#[repr(C)]). + + The callback fills this in. `body` must be allocated with the C + runtime malloc that the native library's free() matches: the + native side takes ownership and frees it on both the success and + the error return path. + """ + _fields_ = [ + ("status", ctypes.c_int), + ("body", ctypes.POINTER(ctypes.c_ubyte)), + ("body_len", ctypes.c_size_t), + ] + + +class C2paHttpResolver(ctypes.Structure): + """Opaque structure for a native HTTP resolver handle.""" + _fields_ = [] # Empty as it's opaque in the C API + + +HttpResolverCallback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_void_p, + ctypes.POINTER(C2paHttpRequest), + ctypes.POINTER(C2paHttpResponse)) + # Helper function to set function prototypes @@ -1046,6 +1095,18 @@ def _setup_function(func, argtypes, restype=None): ctypes.c_int ) +# HTTP resolver bindings +_setup_function( + _lib.c2pa_http_resolver_create, + [ctypes.c_void_p, HttpResolverCallback], + ctypes.POINTER(C2paHttpResolver) +) +_setup_function( + _lib.c2pa_context_builder_set_http_resolver, + [ctypes.POINTER(C2paContextBuilder), ctypes.POINTER(C2paHttpResolver)], + ctypes.c_int +) + _setup_function( _lib.c2pa_error_set_last, [ctypes.c_char_p], @@ -1504,269 +1565,189 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: return mimetypes.guess_type(str(path))[0] or "" -class NetworkResource(ManagedResource): - """Owns a native HTTP resolver handle for a Context. - - Private implementation detail of the custom-resolver feature: not part - of the public API. Nests every native mirror/helper the feature needs - (ctypes structs, the malloc used for response bodies, header parsing, - resolver normalization, and the ctypes trampoline builder) so nothing - about the FFI plumbing is part of the public surface. Context is the - only thing that uses this class. - - This class itself is not re-exported from the top-level c2pa package - or listed in its __all__ -- it's still a real, directly importable - class (`from c2pa.c2pa import NetworkResource`), just excluded from - the curated surface next to Reader/Builder/Context. - - c2pa ships no HttpRequest/HttpResponse/HttpResolver types at all: a - resolver passed to ContextBuilder.with_resolver()/Context(resolver=...) - is never isinstance-checked. Any callable, or any object exposing a - resolve(request) method, is accepted (see _coerce_resolver below); the - request handed to it exposes .url/.method/.headers/.body, and the - value it returns only needs .status (int) and .body (bytes) attributes - -- pure duck typing, both ways. tests/network/http_resolver.py has a - copyable reference implementation of that shape. - """ - - class _Request(ctypes.Structure): - """Mirror of the native C2paHttpRequest (#[repr(C)]). +_NATIVE_MALLOC = None - Read-only view: every pointer borrows native memory that is only - valid for the duration of the resolver callback. Copy anything you - keep. - The string fields are c_char_p so ctypes converts them to bytes on - read. `body` stays a c_ubyte pointer because it is binary, not - NUL-terminated. - """ - _fields_ = [ - ("url", ctypes.c_char_p), - ("method", ctypes.c_char_p), - ("headers", ctypes.c_char_p), - ("body", ctypes.POINTER(ctypes.c_ubyte)), - ("body_len", ctypes.c_size_t), - ] - - class _Response(ctypes.Structure): - """Mirror of the native C2paHttpResponse (#[repr(C)]). - - The callback fills this in. `body` must be allocated with the C - runtime malloc that the native library's free() matches: the - native side takes ownership and frees it on both the success and - the error return path. - """ - _fields_ = [ - ("status", ctypes.c_int), - ("body", ctypes.POINTER(ctypes.c_ubyte)), - ("body_len", ctypes.c_size_t), - ] - - class _Handle(ctypes.Structure): - """Opaque structure for a native HTTP resolver handle.""" - _fields_ = [] # Empty as it's opaque in the C API - - _Callback = ctypes.CFUNCTYPE( - ctypes.c_int, - ctypes.c_void_p, - ctypes.POINTER(_Request), - ctypes.POINTER(_Response)) - - class _RequestView: - """Plain Python view of a decoded native HTTP request. - - Internal plumbing only: the trampoline hands one of these to the - resolver's resolve_fn so it has .url/.method/.headers/.body to - read. Not a public type -- callers never construct or import this, - they just read attributes off the instance they're handed. See - tests/network/http_resolver.py for a copyable reference - HttpRequest with the same shape. - """ - __slots__ = ("url", "method", "headers", "body") +def _get_native_malloc(): + """Return malloc from the C runtime whose free() the native library + calls. - def __init__( - self, url: str, method: str, headers: dict, body: bytes): - self.url = url - self.method = method - self.headers = headers - self.body = body + Resolver response bodies are handed to the native library, which + frees them with the platform libc free(). The allocation must come + from the matching runtime or the free is heap corruption, not a + leak. - def __repr__(self): - return (f"_RequestView(method={self.method!r}, " - f"url={self.url!r}, body_len={len(self.body)})") + Looked up lazily so importing c2pa never fails on an exotic + platform unless the resolver feature is actually used. + """ + global _NATIVE_MALLOC + if _NATIVE_MALLOC is None: + if sys.platform == "win32": + try: + # Rust MSVC targets link the UCRT, so libc::free is + # ucrtbase!free. The legacy msvcrt.dll is a different + # heap. + crt = ctypes.CDLL("ucrtbase") + except OSError: + crt = ctypes.CDLL("msvcrt") + else: + crt = ctypes.CDLL(None) + malloc = crt.malloc + malloc.argtypes = [ctypes.c_size_t] + # Required: the default c_int restype truncates 64-bit pointers. + malloc.restype = ctypes.c_void_p + _NATIVE_MALLOC = malloc + return _NATIVE_MALLOC + + +def _parse_header_lines(raw: str) -> dict: + """Parse the FFI's newline-delimited 'Name: Value' header block. + + The native side always sends a string (empty when there are no + headers), never NULL. Header names arrive lowercased, and repeated + headers are sent as separate lines, so the last occurrence of a + name wins here. + """ + headers = {} + for line in raw.split("\n"): + name, sep, value = line.partition(":") + if sep: + headers[name.strip()] = value.strip() + return headers + + +class _HttpRequestView: + """Plain Python view of a decoded native HTTP request. + + Internal plumbing only: the trampoline hands one of these to the + resolver's resolve_fn so it has .url/.method/.headers/.body to + read. Not a public type -- callers never construct or import this, + they just read attributes off the instance they're handed. See + tests/network/http_resolver.py for a copyable reference HttpRequest + with the same shape. + """ + __slots__ = ("url", "method", "headers", "body") + + def __init__(self, url: str, method: str, headers: dict, body: bytes): + self.url = url + self.method = method + self.headers = headers + self.body = body + + def __repr__(self): + return (f"_HttpRequestView(method={self.method!r}, " + f"url={self.url!r}, body_len={len(self.body)})") + + +def _coerce_resolver(resolver): + """Normalize a resolver into a callable taking an HttpRequest. + + Accepts either an object with a resolve(request) method or a bare + callable with the same signature. c2pa ships no HttpRequest/ + HttpResponse/HttpResolver types at all: the resolver passed to + ContextBuilder.with_resolver()/Context(resolver=...) is never + isinstance-checked. The request handed to it exposes + .url/.method/.headers/.body, and the value it returns only needs + .status (int) and .body (bytes) attributes -- pure duck typing, both + ways. tests/network/http_resolver.py has a copyable reference + implementation of that shape. + """ + resolve_fn = getattr(resolver, "resolve", None) + if callable(resolve_fn): + return resolve_fn + if callable(resolver): + return resolver + raise TypeError( + "HTTP resolver must be callable or provide a resolve() method, " + f"got {type(resolver).__name__}") - _native_malloc = None - @staticmethod - def _get_native_malloc(): - """Return malloc from the C runtime whose free() the native library - calls. +def _make_http_resolver_trampoline(resolve_fn): + """Wrap a Python resolve function into a native C callback. - Resolver response bodies are handed to the native library, which - frees them with the platform libc free(). The allocation must come - from the matching runtime or the free is heap corruption, not a - leak. + Args: + resolve_fn: Callable taking a duck-typed request (.url/.method/ + .headers/.body) and returning a duck-typed response + (.status/.body) -- see _HttpRequestView. - Looked up lazily so importing c2pa never fails on an exotic - platform unless the resolver feature is actually used. - """ - if NetworkResource._native_malloc is None: - if sys.platform == "win32": - try: - # Rust MSVC targets link the UCRT, so libc::free is - # ucrtbase!free. The legacy msvcrt.dll is a different - # heap. - crt = ctypes.CDLL("ucrtbase") - except OSError: - crt = ctypes.CDLL("msvcrt") + Returns: + The HttpResolverCallback object. The caller MUST keep a reference + to it for as long as any native context built from it can run: + the native side holds only a raw function pointer, and letting + the thunk be collected while a context can still call it is + undefined behavior. + """ + def _trampoline(_ctx, request_ptr, response_ptr): + try: + req = request_ptr.contents + # Copy everything out now: these pointers borrow native + # memory that is only valid for the duration of this call. + url = req.url.decode("utf-8", "replace") if req.url else "" + method = (req.method.decode("utf-8", "replace") + if req.method else "") + raw_headers = (req.headers.decode("utf-8", "replace") + if req.headers else "") + body = (ctypes.string_at(req.body, req.body_len) + if (req.body and req.body_len) else b"") + + result = resolve_fn(_HttpRequestView( + url=url, + method=method, + headers=_parse_header_lines(raw_headers), + body=body)) + + payload = result.body or b"" + if not isinstance(payload, (bytes, bytearray)): + raise TypeError( + "resolver response .body must be bytes, got " + f"{type(payload).__name__}") + + response = response_ptr.contents + response.status = int(result.status) + if payload: + length = len(payload) + buf = _get_native_malloc()(length) + if not buf: + _lib.c2pa_error_set_last( + b"Other: HTTP resolver out of memory") + return -1 + ctypes.memmove(buf, bytes(payload), length) + # Ownership handoff: from here the native library frees + # this buffer on BOTH return paths (it copies then + # frees on 0, and frees a leftover body on non-zero). + # Never free it here. + response.body = ctypes.cast( + buf, ctypes.POINTER(ctypes.c_ubyte)) + response.body_len = length else: - crt = ctypes.CDLL(None) - malloc = crt.malloc - malloc.argtypes = [ctypes.c_size_t] - # Required: the default c_int restype truncates 64-bit pointers. - malloc.restype = ctypes.c_void_p - NetworkResource._native_malloc = malloc - return NetworkResource._native_malloc - - @staticmethod - def _parse_header_lines(raw: str) -> dict: - """Parse the FFI's newline-delimited 'Name: Value' header block. - - The native side always sends a string (empty when there are no - headers), never NULL. Header names arrive lowercased, and repeated - headers are sent as separate lines, so the last occurrence of a - name wins here. - """ - headers = {} - for line in raw.split("\n"): - name, sep, value = line.partition(":") - if sep: - headers[name.strip()] = value.strip() - return headers - - @staticmethod - def _coerce_resolver(resolver): - """Normalize a resolver into a plain callable taking an HttpRequest. - - Accepts either an object with a resolve(request) method or a bare - callable with the same signature. - """ - resolve_fn = getattr(resolver, "resolve", None) - if callable(resolve_fn): - return resolve_fn - if callable(resolver): - return resolver - raise TypeError( - "HTTP resolver must be callable or provide a resolve() method, " - f"got {type(resolver).__name__}") - - @classmethod - def _make_trampoline(cls, resolve_fn): - """Wrap a Python resolve function into a native C callback. - - Args: - resolve_fn: Callable taking a duck-typed request (.url/ - .method/.headers/.body) and returning a duck-typed - response (.status/.body) -- see cls._RequestView. - - Returns: - The _Callback object. The caller MUST keep a reference to it - for as long as any native context built from it can run: the - native side holds only a raw function pointer, and letting the - thunk be collected while a context can still call it is - undefined behavior. - """ - def _trampoline(_ctx, request_ptr, response_ptr): + # body and body_len must stay NULL/0 together: the + # native side skips its free when body_len is 0, so a + # non-NULL pointer with a zero length is never freed + # and leaks. + response.body = None + response.body_len = 0 + return 0 + except BaseException as e: # noqa: B036 - must not unwind native + # BaseException on purpose: an exception escaping a ctypes + # callback cannot propagate into Rust, so it would be + # reported as a generic failure with the real cause lost. + # Catching it here (including KeyboardInterrupt) turns it + # into a typed error carrying the actual message. + # + # Setting the error is mandatory, not best-effort: the + # native error slot is thread-local and is NOT cleared + # before the callback runs, so returning non-zero without + # setting it surfaces a stale, unrelated error from an + # earlier call. try: - req = request_ptr.contents - # Copy everything out now: these pointers borrow native - # memory that is only valid for the duration of this call. - url = req.url.decode("utf-8", "replace") if req.url else "" - method = (req.method.decode("utf-8", "replace") - if req.method else "") - raw_headers = (req.headers.decode("utf-8", "replace") - if req.headers else "") - body = (ctypes.string_at(req.body, req.body_len) - if (req.body and req.body_len) else b"") - - result = resolve_fn(cls._RequestView( - url=url, - method=method, - headers=cls._parse_header_lines(raw_headers), - body=body)) - - payload = result.body or b"" - if not isinstance(payload, (bytes, bytearray)): - raise TypeError( - "resolver response .body must be bytes, got " - f"{type(payload).__name__}") - - response = response_ptr.contents - response.status = int(result.status) - if payload: - length = len(payload) - buf = cls._get_native_malloc()(length) - if not buf: - _lib.c2pa_error_set_last( - b"Other: HTTP resolver out of memory") - return -1 - ctypes.memmove(buf, bytes(payload), length) - # Ownership handoff: from here the native library frees - # this buffer on BOTH return paths (it copies then - # frees on 0, and frees a leftover body on non-zero). - # Never free it here. - response.body = ctypes.cast( - buf, ctypes.POINTER(ctypes.c_ubyte)) - response.body_len = length - else: - # body and body_len must stay NULL/0 together: the - # native side skips its free when body_len is 0, so a - # non-NULL pointer with a zero length is never freed - # and leaks. - response.body = None - response.body_len = 0 - return 0 - except BaseException as e: # noqa: B036 - must not unwind native - # BaseException on purpose: an exception escaping a ctypes - # callback cannot propagate into Rust, so it would be - # reported as a generic failure with the real cause lost. - # Catching it here (including KeyboardInterrupt) turns it - # into a typed error carrying the actual message. - # - # Setting the error is mandatory, not best-effort: the - # native error slot is thread-local and is NOT cleared - # before the callback runs, so returning non-zero without - # setting it surfaces a stale, unrelated error from an - # earlier call. - try: - _lib.c2pa_error_set_last( - "Other: Python HTTP resolver failed: {}".format(e) - .encode("utf-8", "replace")) - except BaseException: - pass - return -1 - - return cls._Callback(_trampoline) + _lib.c2pa_error_set_last( + "Other: Python HTTP resolver failed: {}".format(e) + .encode("utf-8", "replace")) + except BaseException: + pass + return -1 - def __init__(self, callback_cb): - super().__init__() - self._create_and_activate( - lambda: _lib.c2pa_http_resolver_create(None, callback_cb), - "Failed to create HTTP resolver") - - -# HTTP resolver bindings -_setup_function( - _lib.c2pa_http_resolver_create, - [ctypes.c_void_p, NetworkResource._Callback], - ctypes.POINTER(NetworkResource._Handle) -) -_setup_function( - _lib.c2pa_context_builder_set_http_resolver, - [ctypes.POINTER(C2paContextBuilder), - ctypes.POINTER(NetworkResource._Handle)], - ctypes.c_int -) + return HttpResolverCallback(_trampoline) class ContextProvider(ABC): @@ -1993,7 +1974,7 @@ def with_resolver( - Do not call c2pa APIs from inside the resolver: reentering the FFI while a call is in flight is undefined. """ - NetworkResource._coerce_resolver(resolver) + _coerce_resolver(resolver) self._resolver = resolver return self @@ -2031,6 +2012,18 @@ def __init__(self): _lib.c2pa_context_builder_new, "Failed to create ContextBuilder") + class _NativeHttpResolver(ManagedResource): + """Short-lived wrapper so the native HTTP resolver handle rides + the normal lifecycle: any failure inside its `with` block frees + it via close() unless a consuming call already took it. + """ + + def __init__(self, callback_cb): + super().__init__() + self._create_and_activate( + lambda: _lib.c2pa_http_resolver_create(None, callback_cb), + "Failed to create HTTP resolver") + def __init__( self, settings: Optional['Settings'] = None, @@ -2074,13 +2067,12 @@ def __init__( check=lambda r: r != 0) if resolver is not None: - resolve_fn = NetworkResource._coerce_resolver(resolver) - callback_cb = NetworkResource._make_trampoline( - resolve_fn) + resolve_fn = _coerce_resolver(resolver) + callback_cb = _make_http_resolver_trampoline(resolve_fn) # Pin the thunk before any native call can capture its # raw function pointer (see _release for why it stays). self._http_resolver_cb = callback_cb - with NetworkResource(callback_cb) as native_res: + with self._NativeHttpResolver(callback_cb) as native_res: native_res._consume_no_replacement( lambda h: _lib.c2pa_context_builder_set_http_resolver( diff --git a/tests/network/README.md b/tests/network/README.md index 5c7921c6..77471614 100644 --- a/tests/network/README.md +++ b/tests/network/README.md @@ -1,4 +1,4 @@ -## Using a custom HTTP resolver +# On custom HTTP resolvers A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. @@ -26,9 +26,9 @@ Two things to note before writing one: ### How a custom resolver works under the hood -1. **The wiring.** `ContextBuilder.with_resolver(resolver)` stores the resolver, validating it eagerly. `Context.__init__` (whether reached via the builder or `Context(resolver=...)` directly) wraps it in `NetworkResource` — a private class in `c2pa.py` that owns the native `C2paHttpResolver` handle via `c2pa_http_resolver_create`, following the same `ManagedResource` lifecycle as `Reader`/`Builder`/`Signer`. +1. **The wiring.** `ContextBuilder.with_resolver(resolver)` stores the resolver, validating it eagerly. `Context.__init__` (whether reached via the builder or `Context(resolver=...)` directly) wraps it in `Context._NativeHttpResolver` — a private nested class in `c2pa.py` that owns the native `C2paHttpResolver` handle via `c2pa_http_resolver_create`, following the same `ManagedResource` lifecycle as `Context._NativeBuilder`/`Reader`/`Builder`/`Signer`. -2. **The trampoline.** Your resolver is wrapped into a ctypes `CFUNCTYPE` callback (`NetworkResource._make_trampoline`). The native side calls this callback for every HTTP request the SDK makes through that `Context` — remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG did:web resolution. The callback decodes the native `C2paHttpRequest` struct into plain Python `str`/`bytes`, calls your `resolve()`, and writes the result back into the native `C2paHttpResponse` struct. +2. **The trampoline.** Your resolver is wrapped into a ctypes `CFUNCTYPE` callback (`_make_http_resolver_trampoline`, a module-private function in `c2pa.py`). The native side calls this callback for every HTTP request the SDK makes through that `Context` — remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG did:web resolution. The callback decodes the native `C2paHttpRequest` struct into plain Python `str`/`bytes`, calls your `resolve()`, and writes the result back into the native `C2paHttpResponse` struct. 3. **Memory and ownership handling** — the part implementers most often get wrong: - A non-empty response body must be allocated with the *same C runtime malloc* the native library's `free()` will use (`ucrtbase` before `msvcrt` on Windows, the process's own libc elsewhere). Mismatched allocators cause heap corruption, not a leak. You don't need to worry about this yourself — the trampoline does the allocation and copy for you from the `bytes` your resolver returns. @@ -43,7 +43,7 @@ The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) test exercise The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) test exercises `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. -The [`test_http_resolver_contract.py`](./test_http_resolver_contract.py) test needs no network: it pins the eager-validation behavior above and that c2pa ships no resolver types at all (`HttpRequest`/`HttpResponse`/`HttpResolver` only exist in `http_resolver.py`, not in `c2pa.c2pa`), while `NetworkResource` — the FFI binding itself — stays importable from `c2pa.c2pa`, just not re-exported from the top-level `c2pa` package. +The [`test_http_resolver_contract.py`](./test_http_resolver_contract.py) test needs no network: it pins the eager-validation behavior above and that c2pa ships no resolver types at all (`HttpRequest`/`HttpResponse`/`HttpResolver` only exist in `http_resolver.py`, not in `c2pa.c2pa`). The debug and cache tests use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when the resolver itself observes a transport failure. Run them with: diff --git a/tests/network/test_http_resolver_contract.py b/tests/network/test_http_resolver_contract.py index 82c8a8f2..9950b83e 100644 --- a/tests/network/test_http_resolver_contract.py +++ b/tests/network/test_http_resolver_contract.py @@ -19,9 +19,6 @@ HttpResolver support, and that c2pa ships no resolver types at all -- HttpRequest/HttpResponse/HttpResolver here come from the local reference implementation (tests/network/http_resolver.py), not from c2pa itself. -NetworkResource is the one piece of this feature that does live in c2pa -(it's the FFI binding), so its "importable but not re-exported" property -is pinned here too. """ import os @@ -34,7 +31,6 @@ import c2pa # noqa: E402 import c2pa.c2pa # noqa: E402 -from c2pa.c2pa import NetworkResource # noqa: E402 class TestResolverValidation(unittest.TestCase): @@ -95,14 +91,6 @@ def test_resolver_contract_types_not_in_c2pa_c2pa(self): "tests/network/http_resolver.py for a reference " "implementation.") - def test_network_resource_importable_but_not_reexported(self): - # NetworkResource is the FFI binding, unlike the resolver contract - # types above -- it stays in c2pa.c2pa, just excluded from - # c2pa.__all__ next to Reader/Builder/Context. - self.assertNotIn("NetworkResource", c2pa.__all__) - self.assertFalse(hasattr(c2pa, "NetworkResource")) - self.assertTrue(NetworkResource) # importable from c2pa.c2pa - if __name__ == "__main__": unittest.main() From 535690fe4594e77ebc02df1feaebbe2b74e62328 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 15:06:37 -0700 Subject: [PATCH 46/67] fix: Refactor 3 --- src/c2pa/c2pa.py | 342 ++++++++++++++++++++-------------------- tests/network/README.md | 6 +- 2 files changed, 178 insertions(+), 170 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 706b7774..0bc11175 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -292,6 +292,40 @@ def _free_native_ptr(ptr): result) return result + _native_malloc = None + + @staticmethod + def _get_native_malloc(): + """Return malloc from the C runtime whose free() the native library + calls. + + Resolver response bodies are handed to the native library, which + frees them with the platform libc free(). The allocation must come + from the matching runtime or the free is heap corruption, not a + leak. + + Looked up lazily so importing c2pa never fails on an exotic + platform unless the resolver feature is actually used. + """ + if ManagedResource._native_malloc is None: + if sys.platform == "win32": + try: + # Rust MSVC targets link the UCRT, so libc::free is + # ucrtbase!free. The legacy msvcrt.dll is a different + # heap. + crt = ctypes.CDLL("ucrtbase") + except OSError: + crt = ctypes.CDLL("msvcrt") + else: + crt = ctypes.CDLL(None) + malloc = crt.malloc + malloc.argtypes = [ctypes.c_size_t] + # Required: the default c_int restype truncates 64-bit + # pointers. + malloc.restype = ctypes.c_void_p + ManagedResource._native_malloc = malloc + return ManagedResource._native_malloc + def _ensure_valid_state(self): """Raise if the resource is closed or uninitialized.""" name = type(self).__name__ @@ -1565,66 +1599,26 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: return mimetypes.guess_type(str(path))[0] or "" -_NATIVE_MALLOC = None - - -def _get_native_malloc(): - """Return malloc from the C runtime whose free() the native library - calls. - - Resolver response bodies are handed to the native library, which - frees them with the platform libc free(). The allocation must come - from the matching runtime or the free is heap corruption, not a - leak. - - Looked up lazily so importing c2pa never fails on an exotic - platform unless the resolver feature is actually used. - """ - global _NATIVE_MALLOC - if _NATIVE_MALLOC is None: - if sys.platform == "win32": - try: - # Rust MSVC targets link the UCRT, so libc::free is - # ucrtbase!free. The legacy msvcrt.dll is a different - # heap. - crt = ctypes.CDLL("ucrtbase") - except OSError: - crt = ctypes.CDLL("msvcrt") - else: - crt = ctypes.CDLL(None) - malloc = crt.malloc - malloc.argtypes = [ctypes.c_size_t] - # Required: the default c_int restype truncates 64-bit pointers. - malloc.restype = ctypes.c_void_p - _NATIVE_MALLOC = malloc - return _NATIVE_MALLOC - - -def _parse_header_lines(raw: str) -> dict: - """Parse the FFI's newline-delimited 'Name: Value' header block. - - The native side always sends a string (empty when there are no - headers), never NULL. Header names arrive lowercased, and repeated - headers are sent as separate lines, so the last occurrence of a - name wins here. - """ - headers = {} - for line in raw.split("\n"): - name, sep, value = line.partition(":") - if sep: - headers[name.strip()] = value.strip() - return headers - - -class _HttpRequestView: - """Plain Python view of a decoded native HTTP request. - - Internal plumbing only: the trampoline hands one of these to the - resolver's resolve_fn so it has .url/.method/.headers/.body to - read. Not a public type -- callers never construct or import this, - they just read attributes off the instance they're handed. See - tests/network/http_resolver.py for a copyable reference HttpRequest - with the same shape. +class HttpRequestView: + """The Python-side view of a decoded native HTTP request, plus the + private machinery that turns a Python resolve_fn into the native + trampoline that constructs instances of this class and calls it. + + Attributes: + url: Absolute request URL. + method: HTTP method ("GET", "POST", ...). + headers: Request headers as a dict. Names are lowercased by the + native layer; when a header repeats, the last value wins. + body: Request body bytes (b"" when there is none). Timestamp + requests POST a body; manifest fetches send none. + + All data is copied out of native memory, so it stays valid after the + resolver call returns. Not re-exported from the top-level c2pa + package: c2pa ships no HttpRequest/HttpResponse/HttpResolver types at + all, a resolver passed to ContextBuilder.with_resolver()/ + Context(resolver=...) is never isinstance-checked (see + _coerce_resolver below). See tests/network/http_resolver.py for a + copyable reference implementation of the request/response shape. """ __slots__ = ("url", "method", "headers", "body") @@ -1635,119 +1629,132 @@ def __init__(self, url: str, method: str, headers: dict, body: bytes): self.body = body def __repr__(self): - return (f"_HttpRequestView(method={self.method!r}, " + return (f"HttpRequestView(method={self.method!r}, " f"url={self.url!r}, body_len={len(self.body)})") + @staticmethod + def _parse_header_lines(raw: str) -> dict: + """Parse the FFI's newline-delimited 'Name: Value' header block. -def _coerce_resolver(resolver): - """Normalize a resolver into a callable taking an HttpRequest. - - Accepts either an object with a resolve(request) method or a bare - callable with the same signature. c2pa ships no HttpRequest/ - HttpResponse/HttpResolver types at all: the resolver passed to - ContextBuilder.with_resolver()/Context(resolver=...) is never - isinstance-checked. The request handed to it exposes - .url/.method/.headers/.body, and the value it returns only needs - .status (int) and .body (bytes) attributes -- pure duck typing, both - ways. tests/network/http_resolver.py has a copyable reference - implementation of that shape. - """ - resolve_fn = getattr(resolver, "resolve", None) - if callable(resolve_fn): - return resolve_fn - if callable(resolver): - return resolver - raise TypeError( - "HTTP resolver must be callable or provide a resolve() method, " - f"got {type(resolver).__name__}") + The native side always sends a string (empty when there are no + headers), never NULL. Header names arrive lowercased, and + repeated headers are sent as separate lines, so the last + occurrence of a name wins here. + """ + headers = {} + for line in raw.split("\n"): + name, sep, value = line.partition(":") + if sep: + headers[name.strip()] = value.strip() + return headers + @staticmethod + def _coerce_resolver(resolver): + """Normalize a resolver into a callable taking an HttpRequest. + + Accepts either an object with a resolve(request) method or a bare + callable with the same signature. The request handed to it + exposes .url/.method/.headers/.body, and the value it returns + only needs .status (int) and .body (bytes) attributes -- pure + duck typing, both ways. + """ + resolve_fn = getattr(resolver, "resolve", None) + if callable(resolve_fn): + return resolve_fn + if callable(resolver): + return resolver + raise TypeError( + "HTTP resolver must be callable or provide a resolve() method, " + f"got {type(resolver).__name__}") -def _make_http_resolver_trampoline(resolve_fn): - """Wrap a Python resolve function into a native C callback. + @classmethod + def _make_trampoline(cls, resolve_fn): + """Wrap a Python resolve function into a native C callback. - Args: - resolve_fn: Callable taking a duck-typed request (.url/.method/ - .headers/.body) and returning a duck-typed response - (.status/.body) -- see _HttpRequestView. + Args: + resolve_fn: Callable taking a duck-typed request (.url/ + .method/.headers/.body) and returning a duck-typed + response (.status/.body). - Returns: - The HttpResolverCallback object. The caller MUST keep a reference - to it for as long as any native context built from it can run: - the native side holds only a raw function pointer, and letting - the thunk be collected while a context can still call it is - undefined behavior. - """ - def _trampoline(_ctx, request_ptr, response_ptr): - try: - req = request_ptr.contents - # Copy everything out now: these pointers borrow native - # memory that is only valid for the duration of this call. - url = req.url.decode("utf-8", "replace") if req.url else "" - method = (req.method.decode("utf-8", "replace") - if req.method else "") - raw_headers = (req.headers.decode("utf-8", "replace") - if req.headers else "") - body = (ctypes.string_at(req.body, req.body_len) - if (req.body and req.body_len) else b"") - - result = resolve_fn(_HttpRequestView( - url=url, - method=method, - headers=_parse_header_lines(raw_headers), - body=body)) - - payload = result.body or b"" - if not isinstance(payload, (bytes, bytearray)): - raise TypeError( - "resolver response .body must be bytes, got " - f"{type(payload).__name__}") - - response = response_ptr.contents - response.status = int(result.status) - if payload: - length = len(payload) - buf = _get_native_malloc()(length) - if not buf: - _lib.c2pa_error_set_last( - b"Other: HTTP resolver out of memory") - return -1 - ctypes.memmove(buf, bytes(payload), length) - # Ownership handoff: from here the native library frees - # this buffer on BOTH return paths (it copies then - # frees on 0, and frees a leftover body on non-zero). - # Never free it here. - response.body = ctypes.cast( - buf, ctypes.POINTER(ctypes.c_ubyte)) - response.body_len = length - else: - # body and body_len must stay NULL/0 together: the - # native side skips its free when body_len is 0, so a - # non-NULL pointer with a zero length is never freed - # and leaks. - response.body = None - response.body_len = 0 - return 0 - except BaseException as e: # noqa: B036 - must not unwind native - # BaseException on purpose: an exception escaping a ctypes - # callback cannot propagate into Rust, so it would be - # reported as a generic failure with the real cause lost. - # Catching it here (including KeyboardInterrupt) turns it - # into a typed error carrying the actual message. - # - # Setting the error is mandatory, not best-effort: the - # native error slot is thread-local and is NOT cleared - # before the callback runs, so returning non-zero without - # setting it surfaces a stale, unrelated error from an - # earlier call. + Returns: + The HttpResolverCallback object. The caller MUST keep a + reference to it for as long as any native context built from + it can run: the native side holds only a raw function + pointer, and letting the thunk be collected while a context + can still call it is undefined behavior. + """ + def _trampoline(_ctx, request_ptr, response_ptr): try: - _lib.c2pa_error_set_last( - "Other: Python HTTP resolver failed: {}".format(e) - .encode("utf-8", "replace")) - except BaseException: - pass - return -1 + req = request_ptr.contents + # Copy everything out now: these pointers borrow native + # memory that is only valid for the duration of this + # call. + url = req.url.decode("utf-8", "replace") if req.url else "" + method = (req.method.decode("utf-8", "replace") + if req.method else "") + raw_headers = (req.headers.decode("utf-8", "replace") + if req.headers else "") + body = (ctypes.string_at(req.body, req.body_len) + if (req.body and req.body_len) else b"") + + result = resolve_fn(cls( + url=url, + method=method, + headers=cls._parse_header_lines(raw_headers), + body=body)) + + payload = result.body or b"" + if not isinstance(payload, (bytes, bytearray)): + raise TypeError( + "resolver response .body must be bytes, got " + f"{type(payload).__name__}") + + response = response_ptr.contents + response.status = int(result.status) + if payload: + length = len(payload) + buf = ManagedResource._get_native_malloc()(length) + if not buf: + _lib.c2pa_error_set_last( + b"Other: HTTP resolver out of memory") + return -1 + ctypes.memmove(buf, bytes(payload), length) + # Ownership handoff: from here the native library frees + # this buffer on BOTH return paths (it copies then + # frees on 0, and frees a leftover body on non-zero). + # Never free it here. + response.body = ctypes.cast( + buf, ctypes.POINTER(ctypes.c_ubyte)) + response.body_len = length + else: + # body and body_len must stay NULL/0 together: the + # native side skips its free when body_len is 0, so a + # non-NULL pointer with a zero length is never freed + # and leaks. + response.body = None + response.body_len = 0 + return 0 + except BaseException as e: # noqa: B036 - must not unwind native + # BaseException on purpose: an exception escaping a ctypes + # callback cannot propagate into Rust, so it would be + # reported as a generic failure with the real cause lost. + # Catching it here (including KeyboardInterrupt) turns it + # into a typed error carrying the actual message. + # + # Setting the error is mandatory, not best-effort: the + # native error slot is thread-local and is NOT cleared + # before the callback runs, so returning non-zero without + # setting it surfaces a stale, unrelated error from an + # earlier call. + try: + _lib.c2pa_error_set_last( + "Other: Python HTTP resolver failed: {}".format(e) + .encode("utf-8", "replace")) + except BaseException: + pass + return -1 - return HttpResolverCallback(_trampoline) + return HttpResolverCallback(_trampoline) class ContextProvider(ABC): @@ -1974,7 +1981,7 @@ def with_resolver( - Do not call c2pa APIs from inside the resolver: reentering the FFI while a call is in flight is undefined. """ - _coerce_resolver(resolver) + HttpRequestView._coerce_resolver(resolver) self._resolver = resolver return self @@ -2067,8 +2074,9 @@ def __init__( check=lambda r: r != 0) if resolver is not None: - resolve_fn = _coerce_resolver(resolver) - callback_cb = _make_http_resolver_trampoline(resolve_fn) + resolve_fn = HttpRequestView._coerce_resolver(resolver) + callback_cb = HttpRequestView._make_trampoline( + resolve_fn) # Pin the thunk before any native call can capture its # raw function pointer (see _release for why it stays). self._http_resolver_cb = callback_cb diff --git a/tests/network/README.md b/tests/network/README.md index 77471614..c0aae78f 100644 --- a/tests/network/README.md +++ b/tests/network/README.md @@ -24,7 +24,7 @@ Two things to note before writing one: - A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. - The SDK does not follow redirects by default. Delegating to `urllib.request` as `http_resolver.py`'s examples do gives you redirect handling for free. -### How a custom resolver works under the hood +## How a custom resolver works 1. **The wiring.** `ContextBuilder.with_resolver(resolver)` stores the resolver, validating it eagerly. `Context.__init__` (whether reached via the builder or `Context(resolver=...)` directly) wraps it in `Context._NativeHttpResolver` — a private nested class in `c2pa.py` that owns the native `C2paHttpResolver` handle via `c2pa_http_resolver_create`, following the same `ManagedResource` lifecycle as `Context._NativeBuilder`/`Reader`/`Builder`/`Signer`. @@ -37,7 +37,7 @@ Two things to note before writing one: - The callback thunk is pinned on the `Context` and deliberately *not* cleared when the `Context` closes: native `Reader`/`Builder` instances clone the underlying Arc, so your resolver can still be invoked by a native clone after `Context.close()`, for as long as that `Reader`/`Builder` is alive. Your resolver must stay valid (and thread-safe — it may be called from SDK worker threads) for that whole lifetime. - Exceptions cannot unwind across the ctypes/native boundary: the trampoline catches every exception your resolver raises, reports it to the native error slot, and turns it into a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch — never from inside your `resolve()` itself. -### The reference tests +### The testable examples The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) test exercises `DebugHttpResolver`: logs the method and URL of each request and the status of each response. @@ -45,7 +45,7 @@ The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) test exercise The [`test_http_resolver_contract.py`](./test_http_resolver_contract.py) test needs no network: it pins the eager-validation behavior above and that c2pa ships no resolver types at all (`HttpRequest`/`HttpResponse`/`HttpResolver` only exist in `http_resolver.py`, not in `c2pa.c2pa`). -The debug and cache tests use `tests/fixtures/cloud.jpg`, which has no embedded manifest, only a remote one, so they need network access; each test skips (rather than fails) when the resolver itself observes a transport failure. Run them with: +Run the testable examples with: ```bash python ./tests/network/test_http_resolver_contract.py # offline-safe From 258c619965306841311dac434ae5995f28b7f9a3 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 15:18:11 -0700 Subject: [PATCH 47/67] fix: Refactor 3 --- src/c2pa/c2pa.py | 110 ++++++++----------- tests/network/README.md | 3 - tests/network/http_resolver.py | 2 +- tests/network/network_test_helpers.py | 2 +- tests/network/test_http_resolver_cache.py | 2 +- tests/network/test_http_resolver_contract.py | 96 ---------------- tests/network/test_http_resolver_debug.py | 2 +- 7 files changed, 51 insertions(+), 166 deletions(-) delete mode 100644 tests/network/test_http_resolver_contract.py diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 0bc11175..65bb0379 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -296,23 +296,16 @@ def _free_native_ptr(ptr): @staticmethod def _get_native_malloc(): - """Return malloc from the C runtime whose free() the native library - calls. - - Resolver response bodies are handed to the native library, which - frees them with the platform libc free(). The allocation must come - from the matching runtime or the free is heap corruption, not a - leak. + """ + Return malloc from the C runtime whose free() is used. - Looked up lazily so importing c2pa never fails on an exotic - platform unless the resolver feature is actually used. + Some allocations must come from the matching runtime + or the free is heap corruption, so we retrieve that here. + Looked up lazily as needed, if needed, """ if ManagedResource._native_malloc is None: if sys.platform == "win32": try: - # Rust MSVC targets link the UCRT, so libc::free is - # ucrtbase!free. The legacy msvcrt.dll is a different - # heap. crt = ctypes.CDLL("ucrtbase") except OSError: crt = ctypes.CDLL("msvcrt") @@ -320,8 +313,7 @@ def _get_native_malloc(): crt = ctypes.CDLL(None) malloc = crt.malloc malloc.argtypes = [ctypes.c_size_t] - # Required: the default c_int restype truncates 64-bit - # pointers. + # The default c_int restype truncates 64-bit pointers. malloc.restype = ctypes.c_void_p ManagedResource._native_malloc = malloc return ManagedResource._native_malloc @@ -845,7 +837,8 @@ class C2paContext(ctypes.Structure): class C2paHttpRequest(ctypes.Structure): - """Matches the native C2paHttpRequest going through the C FFI. + """Matches the native C2paHttpRequest going through the C FFI, + useful if planning to write custom HTTP resolvers. Read-only view: Every pointer borrows native memory that is only valid @@ -867,11 +860,13 @@ class C2paHttpRequest(ctypes.Structure): class C2paHttpResponse(ctypes.Structure): - """Mirror of the native C2paHttpResponse (#[repr(C)]). + """Mirror of the native C2paHttpResponse, + useful if planning to write custom HTTP resolvers. - The callback fills this in. `body` must be allocated with the C - runtime malloc that the native library's free() matches: the - native side takes ownership and frees it on both the success and + The HTTP resolver callback fills this data in. + `body` must be allocated with the C runtime malloc + that the native library's free() matches: the native + side takes ownership and frees it on both the success and the error return path. """ _fields_ = [ @@ -1600,25 +1595,20 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: class HttpRequestView: - """The Python-side view of a decoded native HTTP request, plus the - private machinery that turns a Python resolve_fn into the native + """This Python-side view of a decoded native HTTP request, with the + plumbing that turns a Python resolve_fn into the native trampoline that constructs instances of this class and calls it. Attributes: url: Absolute request URL. method: HTTP method ("GET", "POST", ...). - headers: Request headers as a dict. Names are lowercased by the - native layer; when a header repeats, the last value wins. - body: Request body bytes (b"" when there is none). Timestamp - requests POST a body; manifest fetches send none. + headers: Request headers as a dict. + Names are lowercased by the native layer. + When a header repeats, the last value wins. + body: Request body bytes (b"" when there is none). All data is copied out of native memory, so it stays valid after the - resolver call returns. Not re-exported from the top-level c2pa - package: c2pa ships no HttpRequest/HttpResponse/HttpResolver types at - all, a resolver passed to ContextBuilder.with_resolver()/ - Context(resolver=...) is never isinstance-checked (see - _coerce_resolver below). See tests/network/http_resolver.py for a - copyable reference implementation of the request/response shape. + resolver call returns. """ __slots__ = ("url", "method", "headers", "body") @@ -1637,9 +1627,9 @@ def _parse_header_lines(raw: str) -> dict: """Parse the FFI's newline-delimited 'Name: Value' header block. The native side always sends a string (empty when there are no - headers), never NULL. Header names arrive lowercased, and - repeated headers are sent as separate lines, so the last - occurrence of a name wins here. + headers), never NULL. + Header names arrive lowercased, and repeated headers are sent + as separate lines, so the last occurrence of a name wins here. """ headers = {} for line in raw.split("\n"): @@ -1650,13 +1640,12 @@ def _parse_header_lines(raw: str) -> dict: @staticmethod def _coerce_resolver(resolver): - """Normalize a resolver into a callable taking an HttpRequest. + """Normalize a HTTP resolver into a callable taking an HttpRequest. Accepts either an object with a resolve(request) method or a bare - callable with the same signature. The request handed to it - exposes .url/.method/.headers/.body, and the value it returns - only needs .status (int) and .body (bytes) attributes -- pure - duck typing, both ways. + callable with the same signature. + The request handed to it exposes .url/.method/.headers/.body, + and the value it returns only needs .status (int) and .body (bytes).. """ resolve_fn = getattr(resolver, "resolve", None) if callable(resolve_fn): @@ -1669,7 +1658,7 @@ def _coerce_resolver(resolver): @classmethod def _make_trampoline(cls, resolve_fn): - """Wrap a Python resolve function into a native C callback. + """Wrap a Python HTTP resolver function into a native C callback. Args: resolve_fn: Callable taking a duck-typed request (.url/ @@ -1677,18 +1666,17 @@ def _make_trampoline(cls, resolve_fn): response (.status/.body). Returns: - The HttpResolverCallback object. The caller MUST keep a - reference to it for as long as any native context built from - it can run: the native side holds only a raw function - pointer, and letting the thunk be collected while a context + The HttpResolverCallback object. + The caller MUST keep a reference to it for as long as + any native context built from it can run: + the native side holds only a raw function pointer, + and letting the thunk be collected while a context can still call it is undefined behavior. """ def _trampoline(_ctx, request_ptr, response_ptr): try: req = request_ptr.contents - # Copy everything out now: these pointers borrow native - # memory that is only valid for the duration of this - # call. + # Copy everything out now to keep it. url = req.url.decode("utf-8", "replace") if req.url else "" method = (req.method.decode("utf-8", "replace") if req.method else "") @@ -1720,32 +1708,28 @@ def _trampoline(_ctx, request_ptr, response_ptr): return -1 ctypes.memmove(buf, bytes(payload), length) # Ownership handoff: from here the native library frees - # this buffer on BOTH return paths (it copies then - # frees on 0, and frees a leftover body on non-zero). + # this buffer on return paths. # Never free it here. response.body = ctypes.cast( buf, ctypes.POINTER(ctypes.c_ubyte)) response.body_len = length else: - # body and body_len must stay NULL/0 together: the - # native side skips its free when body_len is 0, so a - # non-NULL pointer with a zero length is never freed - # and leaks. + # body and body_len must stay NULL/0 together: + # the native side skips its free when body_len is 0. response.body = None response.body_len = 0 return 0 except BaseException as e: # noqa: B036 - must not unwind native - # BaseException on purpose: an exception escaping a ctypes - # callback cannot propagate into Rust, so it would be - # reported as a generic failure with the real cause lost. - # Catching it here (including KeyboardInterrupt) turns it - # into a typed error carrying the actual message. + # An exception escaping a ctypes callback cannot propagate + # into Rust, so it would be reported as a generic failure + # with the real cause lost. + # Catching it here turns it into a typed error carrying + # the actual message. # - # Setting the error is mandatory, not best-effort: the - # native error slot is thread-local and is NOT cleared - # before the callback runs, so returning non-zero without - # setting it surfaces a stale, unrelated error from an - # earlier call. + # Setting the error is mandatory: the native error slot is + # thread-local and is not cleared before the callback runs, + # so returning non-zero without setting it could otherwise + # surface a stale error. try: _lib.c2pa_error_set_last( "Other: Python HTTP resolver failed: {}".format(e) diff --git a/tests/network/README.md b/tests/network/README.md index c0aae78f..57f53632 100644 --- a/tests/network/README.md +++ b/tests/network/README.md @@ -43,12 +43,9 @@ The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) test exercise The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) test exercises `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. -The [`test_http_resolver_contract.py`](./test_http_resolver_contract.py) test needs no network: it pins the eager-validation behavior above and that c2pa ships no resolver types at all (`HttpRequest`/`HttpResponse`/`HttpResolver` only exist in `http_resolver.py`, not in `c2pa.c2pa`). - Run the testable examples with: ```bash -python ./tests/network/test_http_resolver_contract.py # offline-safe python ./tests/network/test_http_resolver_debug.py # needs network python ./tests/network/test_http_resolver_cache.py # needs network diff --git a/tests/network/http_resolver.py b/tests/network/http_resolver.py index 88fe8008..9cf3c67a 100644 --- a/tests/network/http_resolver.py +++ b/tests/network/http_resolver.py @@ -1,4 +1,4 @@ -# Copyright 2025 Adobe. All rights reserved. +# Copyright 2026 Adobe. All rights reserved. # This file is licensed to you under the Apache License, # Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) # or the MIT license (http://opensource.org/licenses/MIT), diff --git a/tests/network/network_test_helpers.py b/tests/network/network_test_helpers.py index 25298f9f..a253eccb 100644 --- a/tests/network/network_test_helpers.py +++ b/tests/network/network_test_helpers.py @@ -1,4 +1,4 @@ -# Copyright 2025 Adobe. All rights reserved. +# Copyright 2026 Adobe. All rights reserved. # This file is licensed to you under the Apache License, # Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) # or the MIT license (http://opensource.org/licenses/MIT), diff --git a/tests/network/test_http_resolver_cache.py b/tests/network/test_http_resolver_cache.py index 134deaeb..beab3419 100644 --- a/tests/network/test_http_resolver_cache.py +++ b/tests/network/test_http_resolver_cache.py @@ -1,4 +1,4 @@ -# Copyright 2025 Adobe. All rights reserved. +# Copyright 2026 Adobe. All rights reserved. # This file is licensed to you under the Apache License, # Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) # or the MIT license (http://opensource.org/licenses/MIT), diff --git a/tests/network/test_http_resolver_contract.py b/tests/network/test_http_resolver_contract.py deleted file mode 100644 index 9950b83e..00000000 --- a/tests/network/test_http_resolver_contract.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2025 Adobe. All rights reserved. -# This file is licensed to you under the Apache License, -# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) -# or the MIT license (http://opensource.org/licenses/MIT), -# at your option. - -# Unless required by applicable law or agreed to in writing, -# this software is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or -# implied. See the LICENSE-MIT and LICENSE-APACHE files for the -# specific language governing permissions and limitations under -# each license. - -"""Offline-safe contract tests for the custom HTTP resolver feature. - -No network needed, unlike test_http_resolver_debug.py and -test_http_resolver_cache.py: these pin the resolver validation behavior -(eager TypeError, not deferred to Context.build()), dual bare-callable/ -HttpResolver support, and that c2pa ships no resolver types at all -- -HttpRequest/HttpResponse/HttpResolver here come from the local reference -implementation (tests/network/http_resolver.py), not from c2pa itself. -""" - -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from http_resolver import HttpResolver, HttpResponse # noqa: E402 - -import c2pa # noqa: E402 -import c2pa.c2pa # noqa: E402 - - -class TestResolverValidation(unittest.TestCase): - - def test_with_resolver_rejects_non_callable_eagerly(self): - # Raised by with_resolver() itself, not deferred to build(). - with self.assertRaises(TypeError): - c2pa.Context.builder().with_resolver(42) - - def test_with_resolver_rejects_object_without_resolve_eagerly(self): - with self.assertRaises(TypeError): - c2pa.Context.builder().with_resolver(object()) - - def test_with_resolver_accepts_bare_callable(self): - # Bare callables/duck-typed resolvers are fully supported -- - # c2pa never requires an HttpResolver subclass, or any particular - # class at all. - builder = c2pa.Context.builder().with_resolver( - lambda request: HttpResponse(200, b"")) - self.assertIsInstance(builder, c2pa.ContextBuilder) - - def test_with_resolver_accepts_resolve_method_object(self): - class DuckTypedResolver: - def resolve(self, request): - return HttpResponse(200, b"") - - builder = c2pa.Context.builder().with_resolver(DuckTypedResolver()) - self.assertIsInstance(builder, c2pa.ContextBuilder) - - def test_with_resolver_accepts_http_resolver_subclass(self): - # HttpResolver is the reference implementation's optional base - # class (tests/network/http_resolver.py) -- not required, but - # accepted like anything else with a resolve() method. - class MyResolver(HttpResolver): - def resolve(self, request): - return HttpResponse(200, b"") - - builder = c2pa.Context.builder().with_resolver(MyResolver()) - self.assertIsInstance(builder, c2pa.ContextBuilder) - - def test_context_constructor_rejects_invalid_resolver_eagerly(self): - # Covers the non-builder path: Context(resolver=...) directly. - with self.assertRaises(TypeError): - c2pa.Context(resolver="nope") - - -class TestNoShippedResolverTypes(unittest.TestCase): - """c2pa ships no HttpRequest/HttpResponse/HttpResolver at all: they - aren't reachable from c2pa.c2pa, let alone re-exported from the - top-level c2pa package.""" - - def test_resolver_contract_types_not_in_c2pa_c2pa(self): - for name in ("HttpRequest", "HttpResponse", "HttpResolver"): - self.assertFalse( - hasattr(c2pa.c2pa, name), - f"c2pa.c2pa.{name} should not exist -- the resolver " - "contract is fully duck-typed, with no shipped type. See " - "tests/network/http_resolver.py for a reference " - "implementation.") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/network/test_http_resolver_debug.py b/tests/network/test_http_resolver_debug.py index ea2897bc..2d00747d 100644 --- a/tests/network/test_http_resolver_debug.py +++ b/tests/network/test_http_resolver_debug.py @@ -1,4 +1,4 @@ -# Copyright 2025 Adobe. All rights reserved. +# Copyright 2026 Adobe. All rights reserved. # This file is licensed to you under the Apache License, # Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) # or the MIT license (http://opensource.org/licenses/MIT), From 4dc23fd23b63d438b549628af2affee5d9ac1f13 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:41:55 -0700 Subject: [PATCH 48/67] fix: Renamings --- tests/{network => http_resolver}/README.md | 20 +++++++++---------- tests/{network => http_resolver}/__init__.py | 0 .../http_resolver_example_impl.py} | 8 ++++---- .../http_resolver_test_helpers.py} | 2 +- .../test_http_resolver_cache.py | 8 +++++--- .../test_http_resolver_debug.py | 7 ++++--- 6 files changed, 24 insertions(+), 21 deletions(-) rename tests/{network => http_resolver}/README.md (68%) rename tests/{network => http_resolver}/__init__.py (100%) rename tests/{network/http_resolver.py => http_resolver/http_resolver_example_impl.py} (97%) rename tests/{network/network_test_helpers.py => http_resolver/http_resolver_test_helpers.py} (96%) rename tests/{network => http_resolver}/test_http_resolver_cache.py (95%) rename tests/{network => http_resolver}/test_http_resolver_debug.py (95%) diff --git a/tests/network/README.md b/tests/http_resolver/README.md similarity index 68% rename from tests/network/README.md rename to tests/http_resolver/README.md index 57f53632..b5ad1219 100644 --- a/tests/network/README.md +++ b/tests/http_resolver/README.md @@ -15,20 +15,20 @@ context = c2pa.Context.builder().with_resolver(my_resolver).build() reader = c2pa.Reader("image/jpeg", stream, context=context) ``` -[`http_resolver.py`](./http_resolver.py) in this directory is a reference implementation of that shape — copy it into your own project or adapt it. It has `HttpRequest`, `HttpResponse`, an optional `HttpResolver` base class (subclassing it is not required — it just gets you a documented, type-checkable contract instead of duck typing), plus two example resolvers this test suite exercises: `CachingHttpResolver` (LRU cache with a TTL and retry/backoff for throttled requests) and `DebugHttpResolver` (logs every request/response, delegating the transfer to `urllib`). Neither example imports `c2pa` itself — only the `test_http_resolver_*.py` files do, to exercise them against real `Context`/`Reader`/`Builder` instances. +[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) in this directory is a reference implementation of that shape — copy it into your own project or adapt it. It has `HttpRequest`, `HttpResponse`, an optional `HttpResolver` base class (subclassing it is not required — it just gets you a documented, type-checkable contract instead of duck typing), plus two example resolvers this test suite exercises: `CachingHttpResolver` (LRU cache with a TTL and retry/backoff for throttled requests) and `DebugHttpResolver` (logs every request/response, delegating the transfer to `urllib`). Neither example imports `c2pa` itself — only the `test_http_resolver_*.py` files do, to exercise them against real `Context`/`Reader`/`Builder` instances. Whichever form you use, it's validated immediately: passing something with neither shape raises `TypeError` from `with_resolver()` itself, not later at `.build()`. Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. Two things to note before writing one: - A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. -- The SDK does not follow redirects by default. Delegating to `urllib.request` as `http_resolver.py`'s examples do gives you redirect handling for free. +- The SDK does not follow redirects by default. Delegating to `urllib.request` as `http_resolver_example_impl.py`'s examples do gives you redirect handling for free. ## How a custom resolver works 1. **The wiring.** `ContextBuilder.with_resolver(resolver)` stores the resolver, validating it eagerly. `Context.__init__` (whether reached via the builder or `Context(resolver=...)` directly) wraps it in `Context._NativeHttpResolver` — a private nested class in `c2pa.py` that owns the native `C2paHttpResolver` handle via `c2pa_http_resolver_create`, following the same `ManagedResource` lifecycle as `Context._NativeBuilder`/`Reader`/`Builder`/`Signer`. -2. **The trampoline.** Your resolver is wrapped into a ctypes `CFUNCTYPE` callback (`_make_http_resolver_trampoline`, a module-private function in `c2pa.py`). The native side calls this callback for every HTTP request the SDK makes through that `Context` — remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG did:web resolution. The callback decodes the native `C2paHttpRequest` struct into plain Python `str`/`bytes`, calls your `resolve()`, and writes the result back into the native `C2paHttpResponse` struct. +2. **The trampoline.** Your resolver is wrapped into a ctypes `CFUNCTYPE` callback (`C2paHttpResolverBridge._make_trampoline`, internal to `c2pa.py`). The native side calls this callback for every HTTP request the SDK makes through that `Context` — remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG did:web resolution. The callback decodes the native `C2paHttpRequest` struct into a plain Python `C2paHttpRequestData` holding `str`/`bytes`, calls your `resolve()`, and writes the result back into the native `C2paHttpResponse` struct. 3. **Memory and ownership handling** — the part implementers most often get wrong: - A non-empty response body must be allocated with the *same C runtime malloc* the native library's `free()` will use (`ucrtbase` before `msvcrt` on Windows, the process's own libc elsewhere). Mismatched allocators cause heap corruption, not a leak. You don't need to worry about this yourself — the trampoline does the allocation and copy for you from the `bytes` your resolver returns. @@ -37,18 +37,18 @@ Two things to note before writing one: - The callback thunk is pinned on the `Context` and deliberately *not* cleared when the `Context` closes: native `Reader`/`Builder` instances clone the underlying Arc, so your resolver can still be invoked by a native clone after `Context.close()`, for as long as that `Reader`/`Builder` is alive. Your resolver must stay valid (and thread-safe — it may be called from SDK worker threads) for that whole lifetime. - Exceptions cannot unwind across the ctypes/native boundary: the trampoline catches every exception your resolver raises, reports it to the native error slot, and turns it into a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch — never from inside your `resolve()` itself. -### The testable examples +### Running the test examples -The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) test exercises `DebugHttpResolver`: logs the method and URL of each request and the status of each response. +The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) uses `DebugHttpResolver`: logs the method and URL of each request and the status of each response. -The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) test exercises `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. +The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) uses `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. -Run the testable examples with: +Run the examples with: ```bash -python ./tests/network/test_http_resolver_debug.py # needs network -python ./tests/network/test_http_resolver_cache.py # needs network +python ./tests/http_resolver/test_http_resolver_debug.py +python ./tests/http_resolver/test_http_resolver_cache.py # Or the whole directory via unittest discovery: -python -m unittest discover -s tests/network -v +python -m unittest discover -s tests/http_resolver -v ``` diff --git a/tests/network/__init__.py b/tests/http_resolver/__init__.py similarity index 100% rename from tests/network/__init__.py rename to tests/http_resolver/__init__.py diff --git a/tests/network/http_resolver.py b/tests/http_resolver/http_resolver_example_impl.py similarity index 97% rename from tests/network/http_resolver.py rename to tests/http_resolver/http_resolver_example_impl.py index 9cf3c67a..7997211f 100644 --- a/tests/network/http_resolver.py +++ b/tests/http_resolver/http_resolver_example_impl.py @@ -189,8 +189,8 @@ def _fetch_with_retries(self, request): time.sleep(delay) except urllib.error.URLError as e: # DNS failure, connection refused, timeout: recorded so - # network_test_helpers.skip_if_offline can tell this apart - # from a real bug. + # http_resolver_test_helpers.skip_if_offline can tell this + # apart from a real bug. self.transport_errors.append(e) raise raise RuntimeError("unreachable") @@ -246,7 +246,7 @@ def resolve(self, request): return HttpResponse(e.code, e.read()) except urllib.error.URLError as e: # DNS failure, connection refused, timeout: recorded so - # network_test_helpers.skip_if_offline can tell this apart - # from a real bug. + # http_resolver_test_helpers.skip_if_offline can tell this + # apart from a real bug. self.transport_errors.append(e) raise diff --git a/tests/network/network_test_helpers.py b/tests/http_resolver/http_resolver_test_helpers.py similarity index 96% rename from tests/network/network_test_helpers.py rename to tests/http_resolver/http_resolver_test_helpers.py index a253eccb..2f8a6892 100644 --- a/tests/network/network_test_helpers.py +++ b/tests/http_resolver/http_resolver_test_helpers.py @@ -11,7 +11,7 @@ # specific language governing permissions and limitations under # each license. -"""Shared helpers for the tests/network resolver reference tests. +"""Shared helpers for the tests/http_resolver reference tests. These tests exercise a custom HTTP resolver against the real network (tests/fixtures/cloud.jpg only has a remote manifest, no embedded one). diff --git a/tests/network/test_http_resolver_cache.py b/tests/http_resolver/test_http_resolver_cache.py similarity index 95% rename from tests/network/test_http_resolver_cache.py rename to tests/http_resolver/test_http_resolver_cache.py index beab3419..99c02ef0 100644 --- a/tests/network/test_http_resolver_cache.py +++ b/tests/http_resolver/test_http_resolver_cache.py @@ -11,7 +11,8 @@ # specific language governing permissions and limitations under # each license. -"""Tests for CachingHttpResolver (tests/network/http_resolver.py), +"""Tests for CachingHttpResolver +(tests/http_resolver/http_resolver_example_impl.py), exercised against the real network. Needs internet access to fetch the remote manifest for @@ -26,8 +27,9 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from network_test_helpers import FIXTURES, skip_if_offline # noqa: E402 -from http_resolver import AlwaysFailResolver, CachingHttpResolver # noqa: E402,E501 +from http_resolver_test_helpers import FIXTURES, skip_if_offline # noqa: E402 +from http_resolver_example_impl import ( # noqa: E402 + AlwaysFailResolver, CachingHttpResolver) import c2pa # noqa: E402 diff --git a/tests/network/test_http_resolver_debug.py b/tests/http_resolver/test_http_resolver_debug.py similarity index 95% rename from tests/network/test_http_resolver_debug.py rename to tests/http_resolver/test_http_resolver_debug.py index 2d00747d..e322c577 100644 --- a/tests/network/test_http_resolver_debug.py +++ b/tests/http_resolver/test_http_resolver_debug.py @@ -11,7 +11,8 @@ # specific language governing permissions and limitations under # each license. -"""Tests for DebugHttpResolver (tests/network/http_resolver.py), exercised +"""Tests for DebugHttpResolver +(tests/http_resolver/http_resolver_example_impl.py), exercised against the real network. Demonstrates intercepting every HTTP request the SDK makes through a @@ -28,8 +29,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from network_test_helpers import FIXTURES, skip_if_offline # noqa: E402 -from http_resolver import DebugHttpResolver # noqa: E402 +from http_resolver_test_helpers import FIXTURES, skip_if_offline # noqa: E402 +from http_resolver_example_impl import DebugHttpResolver # noqa: E402 import c2pa # noqa: E402 From 20c0a95756085d9d16c6ba7454216ddf7048e33e Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:50:48 -0700 Subject: [PATCH 49/67] fix: Refactor once more --- src/c2pa/c2pa.py | 62 +++++++---- .../http_resolver_example_impl.py | 14 --- .../http_resolver_test_helpers.py | 46 -------- .../http_resolver/test_http_resolver_cache.py | 79 +++++++------- .../http_resolver/test_http_resolver_debug.py | 103 +++++++++--------- 5 files changed, 132 insertions(+), 172 deletions(-) delete mode 100644 tests/http_resolver/http_resolver_test_helpers.py diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 65bb0379..e8ca1342 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1594,10 +1594,9 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: return mimetypes.guess_type(str(path))[0] or "" -class HttpRequestView: - """This Python-side view of a decoded native HTTP request, with the - plumbing that turns a Python resolve_fn into the native - trampoline that constructs instances of this class and calls it. +class C2paHttpRequestData: + """Python-side decoded copy of a native HTTP request, + handed to a custom HTTP resolver (with_resolve). Attributes: url: Absolute request URL. @@ -1619,9 +1618,15 @@ def __init__(self, url: str, method: str, headers: dict, body: bytes): self.body = body def __repr__(self): - return (f"HttpRequestView(method={self.method!r}, " + return (f"C2paHttpRequestData(method={self.method!r}, " f"url={self.url!r}, body_len={len(self.body)})") + +class C2paHttpResolverBridge: + """Plumbing that turns a Python resolver into the native trampoline + the C API calls. + """ + @staticmethod def _parse_header_lines(raw: str) -> dict: """Parse the FFI's newline-delimited 'Name: Value' header block. @@ -1656,14 +1661,13 @@ def _coerce_resolver(resolver): "HTTP resolver must be callable or provide a resolve() method, " f"got {type(resolver).__name__}") - @classmethod - def _make_trampoline(cls, resolve_fn): + @staticmethod + def _make_trampoline(resolve_fn): """Wrap a Python HTTP resolver function into a native C callback. Args: - resolve_fn: Callable taking a duck-typed request (.url/ - .method/.headers/.body) and returning a duck-typed - response (.status/.body). + resolve_fn: Callable taking a C2paHttpRequestData and + returning a duck-typed response (.status/.body). Returns: The HttpResolverCallback object. @@ -1685,10 +1689,11 @@ def _trampoline(_ctx, request_ptr, response_ptr): body = (ctypes.string_at(req.body, req.body_len) if (req.body and req.body_len) else b"") - result = resolve_fn(cls( + result = resolve_fn(C2paHttpRequestData( url=url, method=method, - headers=cls._parse_header_lines(raw_headers), + headers=C2paHttpResolverBridge._parse_header_lines( + raw_headers), body=body)) payload = result.body or b"" @@ -1697,8 +1702,14 @@ def _trampoline(_ctx, request_ptr, response_ptr): "resolver response .body must be bytes, got " f"{type(payload).__name__}") + status = getattr(result, "status", None) + if not isinstance(status, int) or isinstance(status, bool): + raise TypeError( + "resolver response .status must be an int, got " + f"{type(status).__name__}") + response = response_ptr.contents - response.status = int(result.status) + response.status = status if payload: length = len(payload) buf = ManagedResource._get_native_malloc()(length) @@ -1709,7 +1720,6 @@ def _trampoline(_ctx, request_ptr, response_ptr): ctypes.memmove(buf, bytes(payload), length) # Ownership handoff: from here the native library frees # this buffer on return paths. - # Never free it here. response.body = ctypes.cast( buf, ctypes.POINTER(ctypes.c_ubyte)) response.body_len = length @@ -1881,6 +1891,7 @@ def __init__(self): self._settings = None self._signer = None self._resolver = None + self._resolve_fn = None def with_settings( self, settings: 'Settings', @@ -1925,7 +1936,8 @@ def with_resolver( resolve(request) method, works -- the request it receives exposes .url (str), .method (str), .headers (dict), and .body (bytes); the response it returns only needs .status (int) and .body - (bytes) attributes. See tests/network/http_resolver.py for a + (bytes) attributes. See + tests/http_resolver/http_resolver_example_impl.py for a copyable reference implementation of that shape. Validated immediately: an invalid resolver raises TypeError here, not later at build(). @@ -1965,7 +1977,7 @@ def with_resolver( - Do not call c2pa APIs from inside the resolver: reentering the FFI while a call is in flight is undefined. """ - HttpRequestView._coerce_resolver(resolver) + self._resolve_fn = C2paHttpResolverBridge._coerce_resolver(resolver) self._resolver = resolver return self @@ -1975,6 +1987,8 @@ def build(self) -> 'Context': settings=self._settings, signer=self._signer, resolver=self._resolver, + # Already coerced (and validated) by with_resolver. + _resolve_fn=self._resolve_fn, ) @@ -2020,6 +2034,7 @@ def __init__( settings: Optional['Settings'] = None, signer: Optional['Signer'] = None, resolver=None, + _resolve_fn=None, ): """Create a Context. @@ -2032,6 +2047,9 @@ def __init__( resolve(request) method or a callable with the same signature. See ContextBuilder.with_resolver for the full contract. + _resolve_fn: Private. The already-coerced callable for + `resolver`, passed by ContextBuilder.build() so the + coercion done by with_resolver is not repeated here. Raises: C2paError: If creation fails @@ -2058,11 +2076,15 @@ def __init__( check=lambda r: r != 0) if resolver is not None: - resolve_fn = HttpRequestView._coerce_resolver(resolver) - callback_cb = HttpRequestView._make_trampoline( + # Reuse the builder's coercion when it did one, + # coerce here for the direct Context(resolver=...), + # from_json and from_dict paths. + resolve_fn = _resolve_fn + if resolve_fn is None: + resolve_fn = ( + C2paHttpResolverBridge._coerce_resolver(resolver)) + callback_cb = C2paHttpResolverBridge._make_trampoline( resolve_fn) - # Pin the thunk before any native call can capture its - # raw function pointer (see _release for why it stays). self._http_resolver_cb = callback_cb with self._NativeHttpResolver(callback_cb) as native_res: native_res._consume_no_replacement( diff --git a/tests/http_resolver/http_resolver_example_impl.py b/tests/http_resolver/http_resolver_example_impl.py index 7997211f..05c6a4f2 100644 --- a/tests/http_resolver/http_resolver_example_impl.py +++ b/tests/http_resolver/http_resolver_example_impl.py @@ -153,7 +153,6 @@ def __init__(self, cache=None, timeout=10.0, max_retries=3, self._max_retries = int(max_retries) self._backoff = float(backoff_seconds) self._max_retry_after = float(max_retry_after) - self.transport_errors = [] def resolve(self, request): cacheable = request.method.upper() == "GET" @@ -187,12 +186,6 @@ def _fetch_with_retries(self, request): return e.code, e.read() delay = self._retry_delay(e, attempt) time.sleep(delay) - except urllib.error.URLError as e: - # DNS failure, connection refused, timeout: recorded so - # http_resolver_test_helpers.skip_if_offline can tell this - # apart from a real bug. - self.transport_errors.append(e) - raise raise RuntimeError("unreachable") def _retry_delay(self, error, attempt): @@ -223,7 +216,6 @@ class DebugHttpResolver(HttpResolver): def __init__(self, timeout=10.0): self._timeout = timeout self.requests = [] - self.transport_errors = [] def resolve(self, request): self.requests.append((request.method, request.url)) @@ -244,9 +236,3 @@ def resolve(self, request): # let the SDK turn it into its own typed error: a remote # manifest fetch only accepts 200. return HttpResponse(e.code, e.read()) - except urllib.error.URLError as e: - # DNS failure, connection refused, timeout: recorded so - # http_resolver_test_helpers.skip_if_offline can tell this - # apart from a real bug. - self.transport_errors.append(e) - raise diff --git a/tests/http_resolver/http_resolver_test_helpers.py b/tests/http_resolver/http_resolver_test_helpers.py deleted file mode 100644 index 2f8a6892..00000000 --- a/tests/http_resolver/http_resolver_test_helpers.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2026 Adobe. All rights reserved. -# This file is licensed to you under the Apache License, -# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) -# or the MIT license (http://opensource.org/licenses/MIT), -# at your option. - -# Unless required by applicable law or agreed to in writing, -# this software is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or -# implied. See the LICENSE-MIT and LICENSE-APACHE files for the -# specific language governing permissions and limitations under -# each license. - -"""Shared helpers for the tests/http_resolver reference tests. - -These tests exercise a custom HTTP resolver against the real network -(tests/fixtures/cloud.jpg only has a remote manifest, no embedded one). -""" - -import os -import sys - -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) - -FIXTURES = os.path.join(_REPO_ROOT, "tests", "fixtures") - - -def skip_if_offline(testcase, resolver, exc): - """Skip the test iff the resolver itself observed a transport failure. - - resolver must record urllib.error.URLError instances it catches (DNS - failure, connection refused, timeout) in a `transport_errors` list - before re-raising. Skipping only on an *observed* transport error -- - rather than sniffing exc's message for substrings like "fetch" or - "resolver" -- means a real bug in the resolver or the SDK still fails - the test instead of being silently skipped: a typed error mentioning - "resolver" (the trampoline's own failure-message prefix) is exactly - the kind of bug this distinction is meant to catch. - """ - if resolver.transport_errors: - testcase.skipTest( - "needs internet access to fetch the remote manifest for " - f"tests/fixtures/cloud.jpg: {resolver.transport_errors[-1]}") - raise exc diff --git a/tests/http_resolver/test_http_resolver_cache.py b/tests/http_resolver/test_http_resolver_cache.py index 99c02ef0..f7d4c0ed 100644 --- a/tests/http_resolver/test_http_resolver_cache.py +++ b/tests/http_resolver/test_http_resolver_cache.py @@ -16,8 +16,9 @@ exercised against the real network. Needs internet access to fetch the remote manifest for -tests/fixtures/cloud.jpg; tests skip (not fail) when the resolver itself -observes a transport failure. +tests/fixtures/cloud.jpg: these tests fail without it. On a Python +install with no CA bundle configured, run them with +SSL_CERT_FILE=$(python -m certifi). """ import os @@ -25,9 +26,13 @@ import tempfile import unittest -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(os.path.dirname(_HERE)) +sys.path.insert(0, _HERE) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + +FIXTURES = os.path.join(_REPO_ROOT, "tests", "fixtures") -from http_resolver_test_helpers import FIXTURES, skip_if_offline # noqa: E402 from http_resolver_example_impl import ( # noqa: E402 AlwaysFailResolver, CachingHttpResolver) @@ -45,17 +50,14 @@ def test_read_with_cache(self): expected and worth updating, not a false alarm. """ resolver = CachingHttpResolver() - try: - with c2pa.Context.builder().with_resolver( - resolver).build() as context: - for _ in range(2): - with open(os.path.join(FIXTURES, "cloud.jpg"), - "rb") as f: - with c2pa.Reader( - "image/jpeg", f, context=context) as reader: - reader.get_validation_state() - except c2pa.C2paError as e: - skip_if_offline(self, resolver, e) + with c2pa.Context.builder().with_resolver( + resolver).build() as context: + for _ in range(2): + with open(os.path.join(FIXTURES, "cloud.jpg"), + "rb") as f: + with c2pa.Reader( + "image/jpeg", f, context=context) as reader: + reader.get_validation_state() self.assertEqual(resolver.cache.hits, 1) self.assertEqual(resolver.cache.misses, 1) @@ -109,31 +111,28 @@ def test_sign_with_cache(self): .with_signer(c2pa.Signer.from_info(signer_info)) .build()) try: - try: - builder = c2pa.Builder(manifest_definition, context=context) - - with open(os.path.join(FIXTURES, "cloud.jpg"), - "rb") as ingredient: - for index in range(3): - ingredient.seek(0) - builder.add_ingredient( - {"title": f"cloud.jpg #{index + 1}", - "relationship": "componentOf", - "label": ingredient_labels[index]}, - "image/jpeg", ingredient) - - with tempfile.TemporaryDirectory() as output_dir: - output_path = os.path.join( - output_dir, "A_signed_cached.jpg") - with open(os.path.join(FIXTURES, "A.jpg"), - "rb") as source: - with open(output_path, "wb") as dest: - builder.sign( - c2pa.Signer.from_info(signer_info), - "image/jpeg", source, dest) - self.assertTrue(os.path.exists(output_path)) - except c2pa.C2paError as e: - skip_if_offline(self, resolver, e) + builder = c2pa.Builder(manifest_definition, context=context) + + with open(os.path.join(FIXTURES, "cloud.jpg"), + "rb") as ingredient: + for index in range(3): + ingredient.seek(0) + builder.add_ingredient( + {"title": f"cloud.jpg #{index + 1}", + "relationship": "componentOf", + "label": ingredient_labels[index]}, + "image/jpeg", ingredient) + + with tempfile.TemporaryDirectory() as output_dir: + output_path = os.path.join( + output_dir, "A_signed_cached.jpg") + with open(os.path.join(FIXTURES, "A.jpg"), + "rb") as source: + with open(output_path, "wb") as dest: + builder.sign( + c2pa.Signer.from_info(signer_info), + "image/jpeg", source, dest) + self.assertTrue(os.path.exists(output_path)) finally: context.close() diff --git a/tests/http_resolver/test_http_resolver_debug.py b/tests/http_resolver/test_http_resolver_debug.py index e322c577..7bd89519 100644 --- a/tests/http_resolver/test_http_resolver_debug.py +++ b/tests/http_resolver/test_http_resolver_debug.py @@ -17,8 +17,9 @@ Demonstrates intercepting every HTTP request the SDK makes through a Context. Needs internet access to fetch the remote manifest for -tests/fixtures/cloud.jpg; tests skip (not fail) when the resolver itself -observes a transport failure. +tests/fixtures/cloud.jpg: these tests fail without it. On a Python +install with no CA bundle configured, run them with +SSL_CERT_FILE=$(python -m certifi). """ import json @@ -27,9 +28,13 @@ import tempfile import unittest -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(os.path.dirname(_HERE)) +sys.path.insert(0, _HERE) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + +FIXTURES = os.path.join(_REPO_ROOT, "tests", "fixtures") -from http_resolver_test_helpers import FIXTURES, skip_if_offline # noqa: E402 from http_resolver_example_impl import DebugHttpResolver # noqa: E402 import c2pa # noqa: E402 @@ -41,17 +46,14 @@ def test_read_with_resolver(self): """Reading an asset whose manifest lives at a remote URL logs a GET for that fetch.""" resolver = DebugHttpResolver() - try: - with c2pa.Context.builder().with_resolver( - resolver).build() as context: - with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: - with c2pa.Reader( - "image/jpeg", f, context=context) as reader: - reader.get_validation_state() - self.assertFalse(reader.is_embedded()) - self.assertTrue(reader.get_remote_url()) - except c2pa.C2paError as e: - skip_if_offline(self, resolver, e) + with c2pa.Context.builder().with_resolver( + resolver).build() as context: + with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: + with c2pa.Reader( + "image/jpeg", f, context=context) as reader: + reader.get_validation_state() + self.assertFalse(reader.is_embedded()) + self.assertTrue(reader.get_remote_url()) self.assertTrue( any(method == "GET" for method, _ in resolver.requests)) @@ -98,43 +100,40 @@ def test_sign_with_resolver(self): .with_signer(c2pa.Signer.from_info(signer_info)) .build()) try: - try: - builder = c2pa.Builder(manifest_definition, context=context) - - with open(os.path.join(FIXTURES, "cloud.jpg"), - "rb") as ingredient: - builder.add_ingredient( - {"title": "cloud.jpg", "relationship": "componentOf", - "label": "cloud-ingredient"}, - "image/jpeg", ingredient) - - with tempfile.TemporaryDirectory() as output_dir: - output_path = os.path.join( - output_dir, "A_signed_resolver.jpg") - with open(os.path.join(FIXTURES, "A.jpg"), - "rb") as source: - with open(output_path, "wb") as dest: - builder.sign( - c2pa.Signer.from_info(signer_info), - "image/jpeg", source, dest) - - self.assertTrue( - any(m == "GET" for m, _ in resolver.requests)) - requests_before_reread = len(resolver.requests) - - # Reading the signed file back uses its embedded - # manifest, so this makes no HTTP requests at all. - with open(output_path, "rb") as f: - with c2pa.Reader("image/jpeg", f) as reader: - store = json.loads(reader.json()) - manifest = store["manifests"][ - store["active_manifest"]] - self.assertTrue(manifest.get("ingredients")) - - self.assertEqual( - len(resolver.requests), requests_before_reread) - except c2pa.C2paError as e: - skip_if_offline(self, resolver, e) + builder = c2pa.Builder(manifest_definition, context=context) + + with open(os.path.join(FIXTURES, "cloud.jpg"), + "rb") as ingredient: + builder.add_ingredient( + {"title": "cloud.jpg", "relationship": "componentOf", + "label": "cloud-ingredient"}, + "image/jpeg", ingredient) + + with tempfile.TemporaryDirectory() as output_dir: + output_path = os.path.join( + output_dir, "A_signed_resolver.jpg") + with open(os.path.join(FIXTURES, "A.jpg"), + "rb") as source: + with open(output_path, "wb") as dest: + builder.sign( + c2pa.Signer.from_info(signer_info), + "image/jpeg", source, dest) + + self.assertTrue( + any(m == "GET" for m, _ in resolver.requests)) + requests_before_reread = len(resolver.requests) + + # Reading the signed file back uses its embedded + # manifest, so this makes no HTTP requests at all. + with open(output_path, "rb") as f: + with c2pa.Reader("image/jpeg", f) as reader: + store = json.loads(reader.json()) + manifest = store["manifests"][ + store["active_manifest"]] + self.assertTrue(manifest.get("ingredients")) + + self.assertEqual( + len(resolver.requests), requests_before_reread) finally: context.close() From f449d4f9bc048204bcc1e79d173d87e744544164 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:54:18 -0700 Subject: [PATCH 50/67] fix: Refactor twice more --- src/c2pa/c2pa.py | 16 ++++++++++------ tests/http_resolver/README.md | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index e8ca1342..119134b9 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1595,8 +1595,9 @@ def _get_mime_type_from_path(path: Union[str, Path]) -> str: class C2paHttpRequestData: - """Python-side decoded copy of a native HTTP request, - handed to a custom HTTP resolver (with_resolve). + """Decoded copy of a native HTTP request, + handed to a custom HTTP resolver + (configured using with_resolver on a Context). Attributes: url: Absolute request URL. @@ -1624,7 +1625,8 @@ def __repr__(self): class C2paHttpResolverBridge: """Plumbing that turns a Python resolver into the native trampoline - the C API calls. + the C API calls to handle HTTP resolvers configured on a Context + using `with_resolver`. """ @staticmethod @@ -1719,7 +1721,7 @@ def _trampoline(_ctx, request_ptr, response_ptr): return -1 ctypes.memmove(buf, bytes(payload), length) # Ownership handoff: from here the native library frees - # this buffer on return paths. + # this buffer on return paths. Never free it here. response.body = ctypes.cast( buf, ctypes.POINTER(ctypes.c_ubyte)) response.body_len = length @@ -2076,8 +2078,8 @@ def __init__( check=lambda r: r != 0) if resolver is not None: - # Reuse the builder's coercion when it did one, - # coerce here for the direct Context(resolver=...), + # Reuse the builder's coercion when it did one. + # Coerce here for the direct Context(resolver=...), # from_json and from_dict paths. resolve_fn = _resolve_fn if resolve_fn is None: @@ -2085,6 +2087,8 @@ def __init__( C2paHttpResolverBridge._coerce_resolver(resolver)) callback_cb = C2paHttpResolverBridge._make_trampoline( resolve_fn) + # Pin the thunk before any native call can capture its + # raw function pointer (see _release for why it stays). self._http_resolver_cb = callback_cb with self._NativeHttpResolver(callback_cb) as native_res: native_res._consume_no_replacement( diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index b5ad1219..4f334757 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -43,6 +43,8 @@ The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) uses `DebugHt The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) uses `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. +Both need internet access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED`. This is common with python.org builds on macOS where `Install Certificates.command` was never run; prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. + Run the examples with: ```bash From 184790f13809ad6204b45cbeba9879f8837fe8af Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:00:48 -0700 Subject: [PATCH 51/67] fix: Refactor thrice more --- src/c2pa/c2pa.py | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 119134b9..fe00547b 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1927,22 +1927,8 @@ def with_resolver( """Attach a custom HTTP resolver to the Context being built. The resolver handles every HTTP request the SDK makes through this - Context: remote manifest fetches, OCSP requests, RFC 3161 - timestamp requests, and CAWG did:web resolution. Reader and - Builder instances created without a Context keep using the - built-in resolver. Can be called multiple times; the last - resolver wins. - - c2pa ships no resolver type to subclass or construct: resolver is - never isinstance-checked. Any callable, or any object with a - resolve(request) method, works -- the request it receives exposes - .url (str), .method (str), .headers (dict), and .body (bytes); - the response it returns only needs .status (int) and .body - (bytes) attributes. See - tests/http_resolver/http_resolver_example_impl.py for a - copyable reference implementation of that shape. Validated - immediately: an invalid resolver raises TypeError here, not later - at build(). + Context. Reader and Builder instances created without a Context + keep using the built-in resolver. Args: resolver: A callable, or an object with a resolve(request) @@ -1954,15 +1940,12 @@ def with_resolver( self, for method chaining. Raises: - TypeError: resolver is neither callable nor has a resolve() - method. + TypeError: resolver is neither callable + nor has a resolve() method. Notes: - - Only status 200 is accepted for a remote manifest fetch; any + - Only status 200 is accepted for a remote manifest fetch, any other status surfaces as a typed C2paError. - - The SDK does not follow redirects. A resolver delegating to - urllib.request gets redirect handling for free; a hand-rolled - one must implement it. - Security: a custom resolver bypasses the core.allowed_network_hosts setting, which only filters the built-in resolver. Host filtering becomes the resolver's @@ -1970,14 +1953,8 @@ def with_resolver( - Settings still gate the resolver. With verify.remote_manifest_fetch set to false the resolver is never invoked and the read fails instead. - - Manifests larger than 10 MB are truncated without an error, - because the resolver response carries no Content-Length. - - The resolver may be called from SDK worker threads, so it must - be thread-safe, and it may be called for as long as any Reader - or Builder created from the Context is alive, including after - Context.close(). - - Do not call c2pa APIs from inside the resolver: reentering the - FFI while a call is in flight is undefined. + - Do not call any other c2pa APIs from inside the resolver: + reentering the FFI while a call is in flight is undefined behavior. """ self._resolve_fn = C2paHttpResolverBridge._coerce_resolver(resolver) self._resolver = resolver From b5a2a5f8f1dd3078c1ebab5cc8b5763773b05099 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:15:40 -0700 Subject: [PATCH 52/67] fix: docs --- src/c2pa/c2pa.py | 5 +- tests/http_resolver/README.md | 112 +++++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 26 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index fe00547b..17cc313b 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1997,9 +1997,8 @@ def __init__(self): "Failed to create ContextBuilder") class _NativeHttpResolver(ManagedResource): - """Short-lived wrapper so the native HTTP resolver handle rides - the normal lifecycle: any failure inside its `with` block frees - it via close() unless a consuming call already took it. + """Short-lived wrapper so the potential custom native HTTP + resolver handle (if set) follows the normal lifecycle. """ def __init__(self, callback_cb): diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index 4f334757..09206022 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -1,10 +1,17 @@ -# On custom HTTP resolvers +# Custom HTTP resolvers -A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`. You can use custom resolvers to add headers, cache responses, log traffic, or serve responses from memory in tests. +This document explains how (custom) HTTP resolvers works end to end, what the SDK does and does not do with HTTP, and the platform differences (Linux, Windows, macOS) that affect a resolver in practice. -c2pa ships no resolver types at all — no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `with_resolver()` never does an `isinstance` check: it accepts any callable, or any object with a `resolve(request)` method. The request it receives exposes `.url`, `.method`, `.headers` (dict), and `.body` (bytes); the response it returns only needs `.status` (int) and `.body` (bytes) attributes. +## HTTP resolvers overvie -The minimal form needs no imports at all: +A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`: this lets add headers, cache responses, or serve responses from memory in tests. + +c2pa ships no resolver types at all: there is no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `ContextBuilder.with_resolver()` (and the `Context(resolver=...)` constructor argument, which `Context.from_json`/`from_dict` also accept) never does an `isinstance` check. It accepts either: + +- any callable taking one request argument, or +- any object with a `resolve(request)` method. + +The request handed to you exposes `.url` (str), `.method` (str), `.headers` (dict), and `.body` (bytes). The value you return only needs `.status` (int) and `.body` (bytes) attributes. The minimal form needs no imports at all: ```py def my_resolver(request): @@ -15,37 +22,96 @@ context = c2pa.Context.builder().with_resolver(my_resolver).build() reader = c2pa.Reader("image/jpeg", stream, context=context) ``` -[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) in this directory is a reference implementation of that shape — copy it into your own project or adapt it. It has `HttpRequest`, `HttpResponse`, an optional `HttpResolver` base class (subclassing it is not required — it just gets you a documented, type-checkable contract instead of duck typing), plus two example resolvers this test suite exercises: `CachingHttpResolver` (LRU cache with a TTL and retry/backoff for throttled requests) and `DebugHttpResolver` (logs every request/response, delegating the transfer to `urllib`). Neither example imports `c2pa` itself — only the `test_http_resolver_*.py` files do, to exercise them against real `Context`/`Reader`/`Builder` instances. +The resolver is validated immediately: something with neither shape raises `TypeError` from `with_resolver()` itself (or from the `Context` constructor on the direct path), not later at `.build()`. If a resolver object has both a `resolve()` method and is itself callable, `resolve()` wins. + +[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class — subclassing it is not required, it just gets you a documented, type-checkable contract instead of duck typing — plus three example resolvers: `DebugHttpResolver` (logs every request/response, delegates the transfer to `urllib`), `CachingHttpResolver` (TTL'd LRU cache plus retry/backoff for throttled requests), and `AlwaysFailResolver` (answers every request with a fixed status; needs no network). The module has no dependency on `c2pa` itself — only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. + +## What traffic flows through a resolver (and when) + +A resolver attached to a `Context` handles **every** HTTP request the SDK makes through that `Context`. `Scope and gating to keep in mind: + +- The resolver is **per-Context**. `Reader` and `Builder` instances created *without* that `Context` keep using the SDK's built-in resolver. Attaching a resolver to one `Context` changes nothing anywhere else. +- `with_resolver()` can be called multiple times when creating a context to use: the last resolver set wins and will be used. +- **Settings still gate the resolver.** With `verify.remote_manifest_fetch` set to `false`, the resolver is never invoked for a remote-manifest read — the read fails instead. A resolver is not a way to re-enable disabled fetching. + +## How resolution works, end to end + +Conceptually the pipeline is: native code decides it needs an HTTP resource → it calls back into Python → your resolver performs the transfer however it likes → the response is copied back into native memory. Concretely: + +1. **Wiring.** `Context.__init__` normalizes your resolver into a plain callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver` — the native context builder takes ownership; there is nothing for you to free. +2. **Invocation.** Whenever the native library needs an HTTP resource through that context, it synchronously invokes the trampoline with a pointer to a native request struct and a pointer to a zero-initialized native response struct. The call blocks the SDK operation (`Reader(...)`, `builder.sign(...)`, `add_ingredient(...)`) that triggered it: **there is no SDK-side timeout**. A resolver that hangs, hangs that SDK call. Timeouts are entirely your responsibility (both example resolvers pass `timeout=` to `urllib.request.urlopen`). +3. **Decode.** The trampoline copies everything out of the native request struct — URL, method, headers, body — into a plain Python object (`C2paHttpRequestData`) holding `str`/`bytes`. Because it is a copy, the request object stays valid after your `resolve()` returns; storing it (as `DebugHttpResolver` stores `(method, url)` tuples) is safe. +4. **Resolve.** Your `resolve()` runs and returns a response-shaped object, or raises. +5. **Encode.** The trampoline validates the response (`.status` must be an `int`, `.body` must be `bytes`/`bytearray`), copies a non-empty body into a buffer allocated with the C runtime `malloc`, and writes status/body/length into the native response struct. Ownership of that buffer transfers to native code, which frees it on both the success and the error path. You never allocate or free anything yourself; you only ever return `bytes`. +6. **Consume.** The native side copies the body out, frees the buffer, and hands the response to whatever validation or signing logic asked for it. + +## Request semantics + +- `url` is the absolute request URL. +- `method` is the HTTP verb: `GET` for manifest fetches, `POST` for timestamp requests. +- `headers` is a dict. Header names arrive **lowercased** by the native layer. Repeated headers are delivered as separate lines internally; in the dict, the **last occurrence wins**. +- `body` is `b""` when there is no body (manifest fetches); timestamp requests `POST` a body. `request.body or None` is the idiomatic way to hand it to `urllib.request.Request`, as both examples do. + +## Response and error semantics + +- **Status passes through.** Return the real status; do not translate. For a remote manifest fetch, only `200` is accepted — anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. +- **Raising marks a hard failure.** A transport-level problem (DNS failure, connection refused, timeout) is not a response; raise, and the SDK reports the request as failed. The examples deliberately do *not* catch `urllib.error.URLError` for exactly this reason. +- **Your exception does not propagate as itself.** Exceptions cannot unwind across the ctypes/native boundary. The trampoline catches everything you raise — `BaseException`, including `KeyboardInterrupt` — records its message in the native error slot, and the failure re-emerges as a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch. `except MyCustomError:` around `c2pa.Reader(...)` will never fire; catch `c2pa.C2paError` and read the message. +- **Shape errors are caught early.** Returning a `str` body, a `None` status, or a `bool` status is rejected inside the trampoline with a clear `TypeError` message, which then surfaces the same way (as a `C2paError`). +- An **empty body** is fine: return `b""` (as `AlwaysFailResolver` does), and the trampoline correctly leaves the native body pointer/length pair empty together. + +## Lifetimes + +- **The resolver outlives `Context.close()`.** Native `Reader`/`Builder` instances hold their own reference to the underlying native context, so your resolver can still be invoked after the Python `Context` is closed, for as long as any `Reader` or `Builder` created from it is alive. Do not tear down resolver resources (close a session, release a pool) on `Context.close()`; tie them to the resolvers' own lifetime instead. The SDK internally pins the callback thunk to keep this safe — there is nothing you need to hold onto yourself. +- **No reentrancy.** Do not call c2pa APIs from inside `resolve()`. Re-entering the FFI while a call is in flight is undefined. + +## What the SDK does — and does not do — with HTTP + +The SDK delegates the *transfer* entirely; the resolver *is* the HTTP client. That means: + +- **No redirects.** The SDK does not follow redirects; a `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as both examples do) gives you redirect handling for free; a hand-rolled resolver must implement it. +- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; if you need an allowlist, enforce it yourself in `resolve()` (raise or return an error status for disallowed hosts). +- **TLS is yours.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack your resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives — see below. +- **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. If you serve large manifests, be aware the failure mode downstream is a validation error, not a size error. +- **No caching, retries, or backoff.** Each needed resource is requested; policy is yours. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s — timestamp requests must not be replayed from cache — and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. + +## Platform differences: Linux, Windows, macOS + +The trampoline itself behaves identically everywhere, but four areas differ per platform in ways that bite resolver implementers. + +### 1. The C runtime and the response buffer (why you never allocate) + +The response body buffer must be allocated by the **same C runtime whose `free()` the native library calls** — on the Rust side that is `libc::free`. On Linux and macOS there is effectively one C runtime per process (glibc/musl, libSystem), so "the process's own libc" is always the right allocator. **Windows is different:** a process can host several C runtimes side by side, each with its own heap. Rust's MSVC targets link the Universal CRT, so `free` there is `ucrtbase`'s — while the legacy `msvcrt.dll` is a *different* heap. Allocating from one and freeing into the other is heap corruption, not a leak, and it corrupts silently until it crashes somewhere unrelated. + +The bridge handles this for you: it loads `ucrtbase` first and falls back to `msvcrt`, and does the `malloc`+copy from the `bytes` you return. The rule for implementers is simply: **return `bytes`, never a pointer, never something you allocated with ctypes yourself.** The reason this rule exists is Windows. -Whichever form you use, it's validated immediately: passing something with neither shape raises `TypeError` from `with_resolver()` itself, not later at `.build()`. Raising from the resolver marks the request as a hard failure. Returning a non-200 status passes that status through, and the SDK turns it into a typed `C2paError`. +### 2. TLS trust stores -Two things to note before writing one: +The example resolvers delegate to `urllib`, so they inherit Python's `ssl` defaults — which differ by platform: -- A custom resolver bypasses the `core.allowed_network_hosts` setting, which only filters the built-in resolver. Host filtering becomes your responsibility. -- The SDK does not follow redirects by default. Delegating to `urllib.request` as `http_resolver_example_impl.py`'s examples do gives you redirect handling for free. +- **macOS:** python.org builds do **not** use the system Keychain. If `Install Certificates.command` was never run after installing Python, every HTTPS fetch fails with `CERTIFICATE_VERIFY_FAILED`. Workaround without reinstalling: prefix the run with `SSL_CERT_FILE=$(python -m certifi)`. +- **Linux:** OpenSSL uses the distribution's CA bundle. On a normal desktop this just works; in slim container images the `ca-certificates` package is often missing, producing the same `CERTIFICATE_VERIFY_FAILED` — install the package or set `SSL_CERT_FILE`. +- **Windows:** Python's `ssl` loads roots from the Windows certificate store, so system-managed (including enterprise-injected) roots are honored automatically. Corporate TLS-interception proxies therefore tend to *work* on Windows and *fail* on macOS/Linux with the same code — if a fetch verifies on one machine and not another, compare trust stores before suspecting the resolver. -## How a custom resolver works +A resolver using a different HTTP stack has that stack's trust behavior instead; the point stands that TLS trust is resolver-side, per-platform, and invisible to the SDK. -1. **The wiring.** `ContextBuilder.with_resolver(resolver)` stores the resolver, validating it eagerly. `Context.__init__` (whether reached via the builder or `Context(resolver=...)` directly) wraps it in `Context._NativeHttpResolver` — a private nested class in `c2pa.py` that owns the native `C2paHttpResolver` handle via `c2pa_http_resolver_create`, following the same `ManagedResource` lifecycle as `Context._NativeBuilder`/`Reader`/`Builder`/`Signer`. +### 3. Proxies -2. **The trampoline.** Your resolver is wrapped into a ctypes `CFUNCTYPE` callback (`C2paHttpResolverBridge._make_trampoline`, internal to `c2pa.py`). The native side calls this callback for every HTTP request the SDK makes through that `Context` — remote manifest fetches, OCSP requests, RFC 3161 timestamp requests, and CAWG did:web resolution. The callback decodes the native `C2paHttpRequest` struct into a plain Python `C2paHttpRequestData` holding `str`/`bytes`, calls your `resolve()`, and writes the result back into the native `C2paHttpResponse` struct. +`urllib` discovers proxies differently per platform: environment variables (`http_proxy`, `https_proxy`, `no_proxy`) everywhere, **plus** the Windows registry (Internet Settings) on Windows and the System Configuration framework (Network preferences) on macOS. On Linux, environment variables are the only source. So a resolver built on `urllib` silently follows OS-level proxy settings on Windows/macOS but ignores them on Linux — another way the same resolver code behaves differently per machine. If you need deterministic behavior, configure the proxy (or the absence of one) explicitly in your resolver rather than relying on discovery. -3. **Memory and ownership handling** — the part implementers most often get wrong: - - A non-empty response body must be allocated with the *same C runtime malloc* the native library's `free()` will use (`ucrtbase` before `msvcrt` on Windows, the process's own libc elsewhere). Mismatched allocators cause heap corruption, not a leak. You don't need to worry about this yourself — the trampoline does the allocation and copy for you from the `bytes` your resolver returns. - - Ownership transfers to native code once that buffer is written into the response struct: native frees it on *both* the success path (after copying) and the error path. Nothing on the Python side ever frees it. - - The zero-length trap: an empty body must leave `body`/`body_len` as `NULL`/`0` *together* — a non-NULL pointer with `body_len == 0` is never freed by native code and leaks. The trampoline handles this for you as long as your resolver returns `b""` (not some non-empty placeholder) for an empty body. - - The callback thunk is pinned on the `Context` and deliberately *not* cleared when the `Context` closes: native `Reader`/`Builder` instances clone the underlying Arc, so your resolver can still be invoked by a native clone after `Context.close()`, for as long as that `Reader`/`Builder` is alive. Your resolver must stay valid (and thread-safe — it may be called from SDK worker threads) for that whole lifetime. - - Exceptions cannot unwind across the ctypes/native boundary: the trampoline catches every exception your resolver raises, reports it to the native error slot, and turns it into a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch — never from inside your `resolve()` itself. +### 4. Process model: fork vs. spawn -### Running the test examples +Default `multiprocessing` start methods differ: historically `fork` on Linux (`forkserver` since Python 3.14), `spawn` on macOS and Windows. Under `spawn`, a child process imports fresh and never inherits a `Context` — nothing to think about. Under `fork`, the child inherits the parent's memory image, including a `Context` with a resolver attached. The SDK's native handles are PID-stamped so a forked child neither uses nor frees the parent's native resources (see [`../../docs/native-resources-management.md`](../../docs/native-resources-management.md)); but your *resolver's own* state is ordinary Python and gets copied: locks are cloned in whatever state they were in, background threads do **not** survive the fork, and open sockets/sessions are shared with the parent. A resolver holding only per-call state (like both examples) forks safely; one holding live connections should be recreated in the child. This concern is effectively Linux-only in practice. -The [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) uses `DebugHttpResolver`: logs the method and URL of each request and the status of each response. +## Examples and tests -The [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) uses `CachingHttpResolver`: an LRU cache with a TTL (defaults: 100 items, 120 seconds) that retries throttled requests. Only GET requests answered with 200 are cached. +- [`http_resolver_example_impl.py`](./http_resolver_example_impl.py) — the reference shapes (`HttpRequest`, `HttpResponse`, optional `HttpResolver` ABC) and the three resolvers described above. Copy it into your project and adapt it; it does not import `c2pa`. +- [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) — exercises `DebugHttpResolver`: verifies that a remote-manifest read logs a `GET`, that signing with a remote-manifest ingredient fetches it through the resolver, and that re-reading the signed (embedded-manifest) output performs no HTTP at all. +- [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) — exercises `CachingHttpResolver` (reading twice / ingesting the same ingredient repeatedly hits the cache exactly as the hit/miss counters predict) and `AlwaysFailResolver` (a non-200 answer surfaces as a clean typed `C2paError`, with no network needed). -Both need internet access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED`. This is common with python.org builds on macOS where `Install Certificates.command` was never run; prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. +Both network tests need internet access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` — see the TLS section above; on macOS, prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. -Run the examples with: +Run them in a CLI with the commands: ```bash python ./tests/http_resolver/test_http_resolver_debug.py From 3eea944c208e3f4d35df68b3c5426a4646936d1b Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:16:56 -0700 Subject: [PATCH 53/67] fix: refact0r --- src/c2pa/c2pa.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 17cc313b..986f73ef 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1703,6 +1703,7 @@ def _trampoline(_ctx, request_ptr, response_ptr): raise TypeError( "resolver response .body must be bytes, got " f"{type(payload).__name__}") + data = bytes(payload) status = getattr(result, "status", None) if not isinstance(status, int) or isinstance(status, bool): @@ -1712,14 +1713,14 @@ def _trampoline(_ctx, request_ptr, response_ptr): response = response_ptr.contents response.status = status - if payload: - length = len(payload) + if data: + length = len(data) buf = ManagedResource._get_native_malloc()(length) if not buf: _lib.c2pa_error_set_last( b"Other: HTTP resolver out of memory") return -1 - ctypes.memmove(buf, bytes(payload), length) + ctypes.memmove(buf, data, length) # Ownership handoff: from here the native library frees # this buffer on return paths. Never free it here. response.body = ctypes.cast( @@ -3466,7 +3467,8 @@ def wrapped_callback( return -1 # Copy the signature back to the C buffer - # (since callback is used in native code) + # (since callback is used in native code). + signature = bytes(signature) actual_len = min(len(signature), signed_len) # Use memmove for efficient memory copying instead of # byte-by-byte loop From c785450ca863e90b0bbe57c482e8b46e36a0ddec Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:33:58 -0700 Subject: [PATCH 54/67] fix: docs --- tests/http_resolver/README.md | 58 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index 09206022..bc6f6c69 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -2,11 +2,11 @@ This document explains how (custom) HTTP resolvers works end to end, what the SDK does and does not do with HTTP, and the platform differences (Linux, Windows, macOS) that affect a resolver in practice. -## HTTP resolvers overvie +## HTTP resolvers overview -A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`: this lets add headers, cache responses, or serve responses from memory in tests. +A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`, so you can add headers, cache responses, or serve responses from memory in tests. -c2pa ships no resolver types at all: there is no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `ContextBuilder.with_resolver()` (and the `Context(resolver=...)` constructor argument, which `Context.from_json`/`from_dict` also accept) never does an `isinstance` check. It accepts either: +c2pa ships no resolver types at all: there is no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `ContextBuilder.with_resolver()` (and the `Context(resolver=...)` constructor argument, also accepted by `Context.from_json`/`from_dict`) never does an `isinstance` check. It accepts either: - any callable taking one request argument, or - any object with a `resolve(request)` method. @@ -24,23 +24,23 @@ reader = c2pa.Reader("image/jpeg", stream, context=context) The resolver is validated immediately: something with neither shape raises `TypeError` from `with_resolver()` itself (or from the `Context` constructor on the direct path), not later at `.build()`. If a resolver object has both a `resolve()` method and is itself callable, `resolve()` wins. -[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class — subclassing it is not required, it just gets you a documented, type-checkable contract instead of duck typing — plus three example resolvers: `DebugHttpResolver` (logs every request/response, delegates the transfer to `urllib`), `CachingHttpResolver` (TTL'd LRU cache plus retry/backoff for throttled requests), and `AlwaysFailResolver` (answers every request with a fixed status; needs no network). The module has no dependency on `c2pa` itself — only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. +[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class (subclassing it is not required, it just gets you a documented, type-checkable contract instead of duck typing), plus three example resolvers: `DebugHttpResolver` (logs every request/response, delegates the transfer to `urllib`), `CachingHttpResolver` (TTL'd LRU cache plus retry/backoff for throttled requests), and `AlwaysFailResolver` (answers every request with a fixed status; needs no network). The module has no dependency on `c2pa` itself; only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. ## What traffic flows through a resolver (and when) -A resolver attached to a `Context` handles **every** HTTP request the SDK makes through that `Context`. `Scope and gating to keep in mind: +A resolver attached to a `Context` handles **every** HTTP request the SDK makes through that `Context`. Scope and gating to keep in mind: - The resolver is **per-Context**. `Reader` and `Builder` instances created *without* that `Context` keep using the SDK's built-in resolver. Attaching a resolver to one `Context` changes nothing anywhere else. - `with_resolver()` can be called multiple times when creating a context to use: the last resolver set wins and will be used. -- **Settings still gate the resolver.** With `verify.remote_manifest_fetch` set to `false`, the resolver is never invoked for a remote-manifest read — the read fails instead. A resolver is not a way to re-enable disabled fetching. +- **Settings still gate the resolver.** With `verify.remote_manifest_fetch` set to `false`, the resolver is never invoked for a remote-manifest read; the read fails instead. A resolver is not a way to re-enable disabled fetching. ## How resolution works, end to end -Conceptually the pipeline is: native code decides it needs an HTTP resource → it calls back into Python → your resolver performs the transfer however it likes → the response is copied back into native memory. Concretely: +Conceptually the pipeline is: native code decides it needs an HTTP resource, calls back into Python, your resolver performs the transfer however it likes, then the response is copied back into native memory. Concretely: -1. **Wiring.** `Context.__init__` normalizes your resolver into a plain callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver` — the native context builder takes ownership; there is nothing for you to free. -2. **Invocation.** Whenever the native library needs an HTTP resource through that context, it synchronously invokes the trampoline with a pointer to a native request struct and a pointer to a zero-initialized native response struct. The call blocks the SDK operation (`Reader(...)`, `builder.sign(...)`, `add_ingredient(...)`) that triggered it: **there is no SDK-side timeout**. A resolver that hangs, hangs that SDK call. Timeouts are entirely your responsibility (both example resolvers pass `timeout=` to `urllib.request.urlopen`). -3. **Decode.** The trampoline copies everything out of the native request struct — URL, method, headers, body — into a plain Python object (`C2paHttpRequestData`) holding `str`/`bytes`. Because it is a copy, the request object stays valid after your `resolve()` returns; storing it (as `DebugHttpResolver` stores `(method, url)` tuples) is safe. +1. **Wiring.** `Context.__init__` normalizes your resolver into a plain callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver`; the native context builder takes ownership, so there is nothing for you to free. +2. **Invocation.** Whenever the native library needs an HTTP resource through that context, it synchronously invokes the trampoline with a pointer to a native request struct and a pointer to a zero-initialized native response struct. The call blocks the SDK operation (`Reader(...)`, `builder.sign(...)`, `add_ingredient(...)`) that triggered it, and **there is no SDK-side timeout**. A resolver that hangs, hangs that SDK call. Timeouts are entirely your responsibility (both example resolvers pass `timeout=` to `urllib.request.urlopen`). +3. **Decode.** The trampoline copies everything out of the native request struct (URL, method, headers, body) into a plain Python object (`C2paHttpRequestData`) holding `str`/`bytes`. Because it is a copy, the request object stays valid after your `resolve()` returns; storing it (as `DebugHttpResolver` stores `(method, url)` tuples) is safe. 4. **Resolve.** Your `resolve()` runs and returns a response-shaped object, or raises. 5. **Encode.** The trampoline validates the response (`.status` must be an `int`, `.body` must be `bytes`/`bytearray`), copies a non-empty body into a buffer allocated with the C runtime `malloc`, and writes status/body/length into the native response struct. Ownership of that buffer transfers to native code, which frees it on both the success and the error path. You never allocate or free anything yourself; you only ever return `bytes`. 6. **Consume.** The native side copies the body out, frees the buffer, and hands the response to whatever validation or signing logic asked for it. @@ -54,26 +54,26 @@ Conceptually the pipeline is: native code decides it needs an HTTP resource → ## Response and error semantics -- **Status passes through.** Return the real status; do not translate. For a remote manifest fetch, only `200` is accepted — anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. +- **Status passes through.** Return the real status; do not translate. For a remote manifest fetch, only `200` is accepted; anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. - **Raising marks a hard failure.** A transport-level problem (DNS failure, connection refused, timeout) is not a response; raise, and the SDK reports the request as failed. The examples deliberately do *not* catch `urllib.error.URLError` for exactly this reason. -- **Your exception does not propagate as itself.** Exceptions cannot unwind across the ctypes/native boundary. The trampoline catches everything you raise — `BaseException`, including `KeyboardInterrupt` — records its message in the native error slot, and the failure re-emerges as a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch. `except MyCustomError:` around `c2pa.Reader(...)` will never fire; catch `c2pa.C2paError` and read the message. +- **Your exception does not propagate as itself.** Exceptions cannot unwind across the ctypes/native boundary. The trampoline catches everything you raise, including `BaseException` and `KeyboardInterrupt`, records its message in the native error slot, and the failure re-emerges as a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch. `except MyCustomError:` around `c2pa.Reader(...)` will never fire; catch `c2pa.C2paError` and read the message. - **Shape errors are caught early.** Returning a `str` body, a `None` status, or a `bool` status is rejected inside the trampoline with a clear `TypeError` message, which then surfaces the same way (as a `C2paError`). - An **empty body** is fine: return `b""` (as `AlwaysFailResolver` does), and the trampoline correctly leaves the native body pointer/length pair empty together. ## Lifetimes -- **The resolver outlives `Context.close()`.** Native `Reader`/`Builder` instances hold their own reference to the underlying native context, so your resolver can still be invoked after the Python `Context` is closed, for as long as any `Reader` or `Builder` created from it is alive. Do not tear down resolver resources (close a session, release a pool) on `Context.close()`; tie them to the resolvers' own lifetime instead. The SDK internally pins the callback thunk to keep this safe — there is nothing you need to hold onto yourself. +- **The resolver outlives `Context.close()`.** Native `Reader`/`Builder` instances hold their own reference to the underlying native context, so your resolver can still be invoked after the Python `Context` is closed, for as long as any `Reader` or `Builder` created from it is alive. Do not tear down resolver resources (close a session, release a pool) on `Context.close()`; tie them to the resolvers' own lifetime instead. The SDK internally pins the callback thunk to keep this safe, so there is nothing you need to hold onto yourself. - **No reentrancy.** Do not call c2pa APIs from inside `resolve()`. Re-entering the FFI while a call is in flight is undefined. -## What the SDK does — and does not do — with HTTP +## What the SDK does and does not do with HTTP The SDK delegates the *transfer* entirely; the resolver *is* the HTTP client. That means: -- **No redirects.** The SDK does not follow redirects; a `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as both examples do) gives you redirect handling for free; a hand-rolled resolver must implement it. +- **No redirects.** The SDK does not follow redirects; a `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as both examples do) gives you redirect handling for free. A hand-rolled resolver must implement it. - **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; if you need an allowlist, enforce it yourself in `resolve()` (raise or return an error status for disallowed hosts). -- **TLS is yours.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack your resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives — see below. +- **TLS is yours.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack your resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives; see the platform section below. - **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. If you serve large manifests, be aware the failure mode downstream is a validation error, not a size error. -- **No caching, retries, or backoff.** Each needed resource is requested; policy is yours. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s — timestamp requests must not be replayed from cache — and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. +- **No caching, retries, or backoff.** Each needed resource is requested; policy is yours. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. ## Platform differences: Linux, Windows, macOS @@ -81,35 +81,35 @@ The trampoline itself behaves identically everywhere, but four areas differ per ### 1. The C runtime and the response buffer (why you never allocate) -The response body buffer must be allocated by the **same C runtime whose `free()` the native library calls** — on the Rust side that is `libc::free`. On Linux and macOS there is effectively one C runtime per process (glibc/musl, libSystem), so "the process's own libc" is always the right allocator. **Windows is different:** a process can host several C runtimes side by side, each with its own heap. Rust's MSVC targets link the Universal CRT, so `free` there is `ucrtbase`'s — while the legacy `msvcrt.dll` is a *different* heap. Allocating from one and freeing into the other is heap corruption, not a leak, and it corrupts silently until it crashes somewhere unrelated. +The response body buffer must be allocated by the **same C runtime whose `free()` the native library calls** (on the Rust side that is `libc::free`). On Linux and macOS there is effectively one C runtime per process (glibc/musl, libSystem), so "the process's own libc" is always the right allocator. **Windows is different:** a process can host several C runtimes side by side, each with its own heap. Rust's MSVC targets link the Universal CRT, so `free` there is `ucrtbase`'s, while the legacy `msvcrt.dll` is a *different* heap. Allocating from one and freeing into the other is heap corruption, not a leak, and it corrupts silently until it crashes somewhere unrelated. -The bridge handles this for you: it loads `ucrtbase` first and falls back to `msvcrt`, and does the `malloc`+copy from the `bytes` you return. The rule for implementers is simply: **return `bytes`, never a pointer, never something you allocated with ctypes yourself.** The reason this rule exists is Windows. +The bridge handles this for you: it loads `ucrtbase` first and falls back to `msvcrt`, and does the `malloc`+copy from the `bytes` you return. The rule for implementers: **return `bytes`, never a pointer, never something you allocated with ctypes yourself.** The reason this rule exists is Windows. ### 2. TLS trust stores -The example resolvers delegate to `urllib`, so they inherit Python's `ssl` defaults — which differ by platform: +The example resolvers delegate to `urllib`, so they inherit Python's `ssl` defaults, which differ by platform: - **macOS:** python.org builds do **not** use the system Keychain. If `Install Certificates.command` was never run after installing Python, every HTTPS fetch fails with `CERTIFICATE_VERIFY_FAILED`. Workaround without reinstalling: prefix the run with `SSL_CERT_FILE=$(python -m certifi)`. -- **Linux:** OpenSSL uses the distribution's CA bundle. On a normal desktop this just works; in slim container images the `ca-certificates` package is often missing, producing the same `CERTIFICATE_VERIFY_FAILED` — install the package or set `SSL_CERT_FILE`. -- **Windows:** Python's `ssl` loads roots from the Windows certificate store, so system-managed (including enterprise-injected) roots are honored automatically. Corporate TLS-interception proxies therefore tend to *work* on Windows and *fail* on macOS/Linux with the same code — if a fetch verifies on one machine and not another, compare trust stores before suspecting the resolver. +- **Linux:** OpenSSL uses the distribution's CA bundle. On a normal desktop this just works; in slim container images the `ca-certificates` package is often missing, producing the same `CERTIFICATE_VERIFY_FAILED`. Install the package or set `SSL_CERT_FILE`. +- **Windows:** Python's `ssl` loads roots from the Windows certificate store, so system-managed (including enterprise-injected) roots are honored automatically. Corporate TLS-interception proxies therefore tend to *work* on Windows and *fail* on macOS/Linux with the same code. If a fetch verifies on one machine and not another, compare trust stores before suspecting the resolver. -A resolver using a different HTTP stack has that stack's trust behavior instead; the point stands that TLS trust is resolver-side, per-platform, and invisible to the SDK. +A resolver using a different HTTP stack has that stack's trust behavior instead. TLS trust is resolver-side, per-platform, and invisible to the SDK. ### 3. Proxies -`urllib` discovers proxies differently per platform: environment variables (`http_proxy`, `https_proxy`, `no_proxy`) everywhere, **plus** the Windows registry (Internet Settings) on Windows and the System Configuration framework (Network preferences) on macOS. On Linux, environment variables are the only source. So a resolver built on `urllib` silently follows OS-level proxy settings on Windows/macOS but ignores them on Linux — another way the same resolver code behaves differently per machine. If you need deterministic behavior, configure the proxy (or the absence of one) explicitly in your resolver rather than relying on discovery. +`urllib` discovers proxies differently per platform: environment variables (`http_proxy`, `https_proxy`, `no_proxy`) everywhere, **plus** the Windows registry (Internet Settings) on Windows and the System Configuration framework (Network preferences) on macOS. On Linux, environment variables are the only source. So a resolver built on `urllib` silently follows OS-level proxy settings on Windows/macOS but ignores them on Linux, another way the same resolver code behaves differently per machine. If you need deterministic behavior, configure the proxy (or the absence of one) explicitly in your resolver rather than relying on discovery. ### 4. Process model: fork vs. spawn -Default `multiprocessing` start methods differ: historically `fork` on Linux (`forkserver` since Python 3.14), `spawn` on macOS and Windows. Under `spawn`, a child process imports fresh and never inherits a `Context` — nothing to think about. Under `fork`, the child inherits the parent's memory image, including a `Context` with a resolver attached. The SDK's native handles are PID-stamped so a forked child neither uses nor frees the parent's native resources (see [`../../docs/native-resources-management.md`](../../docs/native-resources-management.md)); but your *resolver's own* state is ordinary Python and gets copied: locks are cloned in whatever state they were in, background threads do **not** survive the fork, and open sockets/sessions are shared with the parent. A resolver holding only per-call state (like both examples) forks safely; one holding live connections should be recreated in the child. This concern is effectively Linux-only in practice. +Default `multiprocessing` start methods differ: historically `fork` on Linux (`forkserver` since Python 3.14), `spawn` on macOS and Windows. Under `spawn`, a child process imports fresh and never inherits a `Context`, so there is nothing to think about. Under `fork`, the child inherits the parent's memory image, including a `Context` with a resolver attached. The SDK's native handles are PID-stamped so a forked child neither uses nor frees the parent's native resources (see [`../../docs/native-resources-management.md`](../../docs/native-resources-management.md)), but your *resolver's own* state is ordinary Python and gets copied: locks are cloned in whatever state they were in, background threads do **not** survive the fork, and open sockets/sessions are shared with the parent. A resolver holding only per-call state (like both examples) forks safely; one holding live connections should be recreated in the child. This concern is Linux-only in practice. ## Examples and tests -- [`http_resolver_example_impl.py`](./http_resolver_example_impl.py) — the reference shapes (`HttpRequest`, `HttpResponse`, optional `HttpResolver` ABC) and the three resolvers described above. Copy it into your project and adapt it; it does not import `c2pa`. -- [`test_http_resolver_debug.py`](./test_http_resolver_debug.py) — exercises `DebugHttpResolver`: verifies that a remote-manifest read logs a `GET`, that signing with a remote-manifest ingredient fetches it through the resolver, and that re-reading the signed (embedded-manifest) output performs no HTTP at all. -- [`test_http_resolver_cache.py`](./test_http_resolver_cache.py) — exercises `CachingHttpResolver` (reading twice / ingesting the same ingredient repeatedly hits the cache exactly as the hit/miss counters predict) and `AlwaysFailResolver` (a non-200 answer surfaces as a clean typed `C2paError`, with no network needed). +- [`http_resolver_example_impl.py`](./http_resolver_example_impl.py): the reference shapes (`HttpRequest`, `HttpResponse`, optional `HttpResolver` ABC) and the three resolvers described above. Copy it into your project and adapt it; it does not import `c2pa`. +- [`test_http_resolver_debug.py`](./test_http_resolver_debug.py): exercises `DebugHttpResolver`, verifying that a remote-manifest read logs a `GET`, that signing with a remote-manifest ingredient fetches it through the resolver, and that re-reading the signed (embedded-manifest) output performs no HTTP at all. +- [`test_http_resolver_cache.py`](./test_http_resolver_cache.py): exercises `CachingHttpResolver` (reading twice / ingesting the same ingredient repeatedly hits the cache exactly as the hit/miss counters predict) and `AlwaysFailResolver` (a non-200 answer surfaces as a clean typed `C2paError`, with no network needed). -Both network tests need internet access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` — see the TLS section above; on macOS, prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. +Both network tests need internet access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above); on macOS, prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. Run them in a CLI with the commands: From 377cd27544dfae2db349ff8edb5b4722526bb8cf Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:31:08 -0700 Subject: [PATCH 55/67] fix: hardening --- pr303-adversarial-review-memory-callbacks.md | 97 +++++++++++++++++++ src/c2pa/c2pa.py | 28 ++++-- tests/http_resolver/README.md | 7 +- .../http_resolver_example_impl.py | 12 +++ 4 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 pr303-adversarial-review-memory-callbacks.md diff --git a/pr303-adversarial-review-memory-callbacks.md b/pr303-adversarial-review-memory-callbacks.md new file mode 100644 index 00000000..8e699ce9 --- /dev/null +++ b/pr303-adversarial-review-memory-callbacks.md @@ -0,0 +1,97 @@ +# PR #303 — adversarial review: memory, callback, and resolver handling + +**Branch state reviewed:** `mathern/http-resolver-custom` at `c785450`. +**Method:** every claim below was adversarially confirmed — against the c2pa-rs FFI source at the pinned tag `c2pa-v0.90.1` (`c2pa_c_ffi/src/c_api.rs`), and where ctypes semantics decide the outcome, by running the experiment (results quoted). The exposed API is untouched by this plan, per instruction. Implementer: Opus 5. + +--- + +## Part A — Confirmed sound. Do not "fix" these. + +Each of these was attacked and held. They are listed so the implementer doesn't churn them. + +**A1. The bytearray snapshot fix (`data = bytes(payload)`) is complete and correct.** Before `184790f`, `length = len(payload)` was computed on the live object and `bytes(payload)` was taken later at `memmove` time; a resolver returning a `bytearray` and shrinking it from another thread in between would have made `memmove` read `length` bytes from a shorter source — an out-of-bounds read. The current code snapshots once and derives `length` from the snapshot. The parallel fix in the signer callback (`signature = bytes(signature)`) closes the same window there. Verified: `bytes(bytearray)` produces an immutable copy. + +**A2. The zero-length body invariant matches Rust exactly.** On the success path Rust skips its `free` when `body.is_null() || body_len == 0`, so a non-NULL pointer with zero length would leak; the trampoline writes `body = None` / `body_len = 0` together. On the error path (`rc != 0`) Rust frees any non-null body unconditionally — and on the trampoline's only mid-write failure (malloc returning NULL), `status` may already be written but `body` is still NULL from Rust's zero-init (`C2paHttpResponse { status: 0, body: null, body_len: 0 }` is constructed fresh per call), so nothing dangles and nothing leaks. + +**A3. The allocator-matching logic is right for every shipped artifact.** Rust frees with `libc::free`; `ucrtbase`-before-`msvcrt` matches Rust MSVC targets, and `CDLL(None)` matches on Unix. Checked `build-wheel.yml`: wheels are `manylinux_2_28` x86_64/aarch64 (glibc) plus mac/Windows — no musllinux target, so there is no shipped configuration where `CDLL(None)` and the native library disagree (see B8 for the residual source-build note). The lazy-init race on `_native_malloc` is benign (idempotent assignment of equivalent objects). + +**A4. The thunk keep-alive chain holds, including failure paths.** `_http_resolver_cb` is pinned on the Context before any native call can capture the raw pointer; `Context._release` deliberately does not clear it; `Reader`/`Builder` store `_context` on construction and clear it only in their own release, at which point their native Arc clone is already being dropped. If `Context.__init__` fails after the resolver was set (signer rejection, build failure), the `with self._NativeBuilder()` block frees the native builder, which owns the resolver by then (`set_http_resolver` does `Box::from_raw` — verified), so no native user of the thunk survives the exception. `_NativeHttpResolver` failure before consumption is freed by its own `with` block. No leak, no dangle, in any ordering I could construct under the documented single-owner usage rules. + +**A5. No uncollectable cycles, no error-slot cross-thread confusion.** The thunk's closure chain (`Context → CFUNCTYPE → _trampoline → resolve_fn → resolver`) has no back-edge to Context unless the user creates one, and PEP 442 collects cycles through `__del__` anyway. `c2pa_error_set_last` writes a thread-local slot and the callback is synchronous, so the error is always set on exactly the thread that reads it after `rc != 0`. + +**A6. Header parsing is injection-safe.** Rust builds the block from `http::HeaderMap` (names already lowercase, values cannot contain `\n` — `to_str()` filters), joins with `\n`, and always sends a valid, possibly empty, never-NULL CString. `partition(":")` keeps colons inside values. The Python side's NULL-guard is defensively redundant, which is fine. + +--- + +## Part B — Findings. Ordered by severity; each with its adversarial confirmation and the fix Opus 5 should implement. + +### B1 (High, security) — The reference resolvers are SSRF/local-file-read gadgets + +The request URL a resolver receives originates from **untrusted asset content**: a remote-manifest reference embedded in whatever JPEG someone hands the application. Both `DebugHttpResolver` and `CachingHttpResolver` pass `request.url` straight to `urllib.request.urlopen`, whose default opener chain includes `FileHandler`, `FTPHandler`, and `DataHandler`. Confirmed consequences: + +- A crafted asset with a `file:///etc/hostname`-style manifest URL makes the resolver read a **local file** and feed its contents into the SDK (and, for `CachingHttpResolver`, into the cache; for `DebugHttpResolver`, the URL into its log). +- `http://169.254.169.254/...` or any internal-network URL is fetched **from inside the caller's network** — the classic SSRF the built-in resolver's `core.allowed_network_hosts` exists to prevent, and which a custom resolver explicitly bypasses. +- `data:` URLs let the asset author synthesize the "fetched" bytes with no network at all. + +The built-in Rust resolver speaks only http/https; the examples silently widen that. Since this directory is the copy-paste reference implementation, the examples are the spec — they must model the safe pattern. + +**Fix:** in `http_resolver_example_impl.py`, both network-delegating resolvers reject non-`http(s)` schemes up front (parse with `urllib.parse.urlsplit`; on any other scheme, raise — a hard failure is the right semantic for "this URL should never be fetched"). Add a short "the URL is attacker-influenced" paragraph to `tests/http_resolver/README.md`'s host-filtering bullet, naming `file://` and internal-host SSRF explicitly. Add a no-network test: build a request-shaped object with a `file://` URL, call `resolve()` directly, assert it raises without touching the filesystem. + +### B2 (Medium, data integrity) — Response status is silently double-truncated; a non-200 can alias to 200 + +Two modular reductions sit between the resolver's return value and what validation sees. Confirmed empirically on the Python side: assigning `2**35 + 200` to a `c_int` struct field **does not raise — it stores 200**. Confirmed in Rust source: `Response::builder().status(resp.status as u16)` truncates i32 → u16. Net effect: status `65736` (or `2**32 + 200`, etc.) reaches validation as a clean `200`; a negative status wraps. A malicious resolver gains nothing (it could return 200 honestly), so this is not an escalation — but a *buggy* resolver (an off-by-arithmetic on a retry counter mixed into status, a status parsed from a corrupt upstream) has its error laundered into success, which for a remote-manifest fetch means "manifest accepted" instead of "fetch failed". Silent modular arithmetic in a validation pipeline is corruption by another name. + +**Fix:** in the trampoline, after the existing int/bool check: `if not 100 <= status <= 599: raise TypeError(f"resolver response .status out of range: {status}")`. The existing `BaseException` handler converts it into a typed error carrying that message. Test: resolver returning `65736` produces a `C2paError` mentioning the out-of-range value, not a success. + +### B3 (Medium, contract integrity) — Falsy non-bytes bodies are silently masked to empty + +`payload = result.body or b""` runs **before** the isinstance check, so `result.body = 0`, `False`, `0.0`, or `[]` all become `b""` and pass — confirmed: `0 or b""` → `b""`. The check that exists specifically to catch wrong-typed bodies has a falsy-shaped hole in it; a resolver bug returning `0` (say, a misplaced status) yields a well-formed empty 200 response instead of a diagnostic. Same family as B2: an error becomes silently valid data. + +**Fix:** replace with explicit None-handling: `payload = result.body; if payload is None: payload = b""` then the existing isinstance check (which now sees `0` and raises). Test: resolver whose response has `body = 0` produces a typed error naming `int`, and `body = None` still means empty body. + +### B4 (Medium, correctness/robustness) — The signer callback lags the trampoline's hardening on three counts + +The resolver trampoline and the signer callback are the two Python→native callbacks in the file; the review compared them line by line. The signer's `wrapped_callback`: + +1. **Catches `Exception`, not `BaseException`.** A `KeyboardInterrupt`/`SystemExit` raised inside the user's signing callback is swallowed by ctypes' own callback guard (ctypes never unwinds into native code — it reports the exception and returns the restype default, `0`). `0` is neither the `-1` error convention nor a plausible signature length; whether native code treats "0 bytes signed" as failure is left to chance, and the real cause is lost. The trampoline's `BaseException` catch exists for exactly this; mirror it. +2. **Never calls `c2pa_error_set_last`.** By the bridge's own (verified) analysis, the native error slot is thread-local and *not cleared* before callbacks — returning `-1` without setting it can surface a **stale error from an earlier call** as the signing failure's diagnosis. The trampoline sets the slot on every failure path; the signer sets it on none. +3. **Silently truncates oversized signatures.** `actual_len = min(len(signature), signed_len)` — a signature longer than the native buffer is cut and returned as if complete, guaranteeing a baffling downstream validation failure instead of a clear local error. Truncated cryptographic material is corruption, not accommodation. + +**Fix:** catch `BaseException`; on every failure path call `c2pa_error_set_last` with a `"Other: Python signer callback failed: ..."` message (same sanitization as B5) before returning `-1`; and replace the `min()` with `if len(signature) > signed_len: set error slot; return -1`. Keep the existing `-1` returns for the input-validation branches but add slot messages there too. Test (unit-level, no signing round-trip needed): a callback returning an oversized `bytes` yields a typed error, not a truncated signature. + +### B5 (Low, diagnostics) — Error-slot messages are NUL-truncatable and unbounded + +`"Other: Python HTTP resolver failed: {}".format(e).encode("utf-8", "replace")` — confirmed empirically that a `bytes` argument with an embedded NUL passes through `c_char_p` fine and the C side simply stops at the NUL. Exception text is partly attacker-influenceable (server-supplied strings inside `URLError`/`HTTPError` messages), so a hostile upstream can blank out the diagnostic tail. Separately, nothing caps the size: a pathological exception message makes Rust build an equally pathological `CString`. Neither is memory-unsafe (that's the point of the confirmation); both degrade the one artifact you need when debugging a resolver in production. + +**Fix:** one tiny helper used by both callbacks (B4 makes the signer need it too): take `str(e)`, replace `"\x00"` with `"\\x00"`, truncate to ~1 KB with an ellipsis marker, then encode. The `"Other: "` prefix is already fixed-position, so error-type spoofing via a crafted message (`"Signature: ..."`) is not possible — confirmed against Rust's first-colon parse; keep the prefix exactly as is. + +### B6 (Low, hardening) — The `rc=0`-with-partial-response hazard is one careless edit away + +The trampoline's safety story depends on a structural invariant nobody wrote down: **no fallible statement may sit between the first write into the response struct and `return 0`.** Today that's true (status write → body write → return). If a future edit inserts anything that can raise after `response.status = status`, the `BaseException` handler returns `-1` and Rust's error path cleans up correctly (frees a body if one was written) — that part is safe. The nastier variant is an edit that makes the *ctypes guard itself* the returner (an exception the handler can't run for, e.g. during interpreter teardown): ctypes returns the restype default `0`, and Rust reads a half-written struct as success. Can't be triggered today; cheap to fence off forever. + +**Fix:** a loud comment at the write site stating the invariant, and — the actually load-bearing part — reorder so `response.status` is written **last**, after the body fields. Then any hypothetical partial state is "body set, status still 0", which Rust's `Response::builder().status(0)` rejects as an invalid status code rather than accepting. Zero-cost, converts the worst theoretical outcome from silent-success to clean failure. + +### B7 (Low, optional hardening) — Reentrancy is documented UB; it could be a clean error instead + +"Do not call c2pa APIs from inside the resolver" is currently enforced by hoping. A thread-local `_in_native_callback` flag set around the `resolve_fn` invocation (and the signer callback body), checked by `_ensure_valid_state` / the few module-level entry points, converts a re-entrant call into an immediate `C2paError("c2pa APIs must not be called from inside a resolver/signer callback")` on the exact offending line. Cost: one thread-local read on the hot path. Recommended, but genuinely optional — decline it if the entry-point audit gets invasive. + +### B8 (Notes; docstring-only, no renames — API surface stays as instructed) + +- The ctypes structs `C2paHttpRequest` / `C2paHttpResponse` / `C2paHttpResolver` carry docstrings saying they're "useful if planning to write custom HTTP resolvers." They aren't — resolver authors never touch ctypes; these are trampoline internals. Since the exposed API must not change, fix only the docstrings (e.g. "Internal ctypes mirror of the native struct; resolver authors interact with the decoded `C2paHttpRequestData` instead"). +- Source builds on musl (Alpine) are the one configuration where `CDLL(None)` could disagree with the native library's allocator if someone produces a static-musl `.so`. Not reachable via shipped wheels (A3); one sentence in the build docs ("native artifact and Python must share a C runtime") closes it. +- Concurrent `close()` while another thread is mid-operation on the same object is a pre-existing, class-wide use-after-free hazard of `ManagedResource` (no in-flight guard exists), not introduced by this PR; the resolver merely adds the thunk to what dangles. Out of scope here — but the resolver docs' thread-safety note should say "do not close objects that another thread is still using" explicitly, since resolver users are the most likely to be multi-threaded. + +--- + +## Part C — Test additions (all no-network, deterministic) + +1. B1: `file://` URL → both example resolvers raise; nothing read from disk (assert via a path that would exist). +2. B2: status `65736` and status `-1` → typed `C2paError`, message names the value. +3. B3: body `0` → typed error naming `int`; body `None` → succeeds as empty. +4. B4: signer callback returning oversized signature → typed error, no truncation; signer callback raising `KeyboardInterrupt` → `-1` path with slot message (invoke `wrapped_callback` directly with ctypes buffers). +5. Regression for A1: resolver returns a `bytearray` and mutates it from a timer thread immediately after `resolve()` returns; signed/validated result must be built from the snapshot (deterministic version: mutate in a subclass's `__buffer__`-free wrapper before returning is enough to prove the copy point — keep it simple: assert the trampoline's copy equals the pre-mutation content by round-tripping through `AlwaysFailResolver`-style in-memory serving). +6. B5: resolver raising `Exception("boom\x00hidden")` → resulting `C2paError` message contains `boom` and the escaped marker, not a truncation at the NUL. + +## Part D — Implementation order + +B1 (examples + README + test) → B2/B3 together (same function, same test file) → B5 helper → B4 signer (uses the helper) → B6 reorder+comment → B7 if clean → B8 docstrings. Each step is independently landable; nothing here touches the exposed API, the FFI signatures, or the lifecycle machinery verified in Part A. diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 986f73ef..95f00b5c 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -837,8 +837,7 @@ class C2paContext(ctypes.Structure): class C2paHttpRequest(ctypes.Structure): - """Matches the native C2paHttpRequest going through the C FFI, - useful if planning to write custom HTTP resolvers. + """Internal ctypes for the native C2paHttpRequest struct. Read-only view: Every pointer borrows native memory that is only valid @@ -860,8 +859,7 @@ class C2paHttpRequest(ctypes.Structure): class C2paHttpResponse(ctypes.Structure): - """Mirror of the native C2paHttpResponse, - useful if planning to write custom HTTP resolvers. + """Internal ctypes mirror of the native C2paHttpResponse struct. The HTTP resolver callback fills this data in. `body` must be allocated with the C runtime malloc @@ -1645,6 +1643,7 @@ def _parse_header_lines(raw: str) -> dict: headers[name.strip()] = value.strip() return headers + @staticmethod def _coerce_resolver(resolver): """Normalize a HTTP resolver into a callable taking an HttpRequest. @@ -1698,7 +1697,9 @@ def _trampoline(_ctx, request_ptr, response_ptr): raw_headers), body=body)) - payload = result.body or b"" + payload = result.body + if payload is None: + payload = b"" if not isinstance(payload, (bytes, bytearray)): raise TypeError( "resolver response .body must be bytes, got " @@ -1710,9 +1711,15 @@ def _trampoline(_ctx, request_ptr, response_ptr): raise TypeError( "resolver response .status must be an int, got " f"{type(status).__name__}") + if not 100 <= status <= 599: + raise TypeError( + "resolver response .status out of range: " + f"{status}") + # No fallible statement between the first write + # into `response` and `return 0`. + # `status` is written last, after the body field. response = response_ptr.contents - response.status = status if data: length = len(data) buf = ManagedResource._get_native_malloc()(length) @@ -1731,10 +1738,11 @@ def _trampoline(_ctx, request_ptr, response_ptr): # the native side skips its free when body_len is 0. response.body = None response.body_len = 0 + response.status = status return 0 except BaseException as e: # noqa: B036 - must not unwind native # An exception escaping a ctypes callback cannot propagate - # into Rust, so it would be reported as a generic failure + # into native lib, so it would be reported as a generic failure # with the real cause lost. # Catching it here turns it into a typed error carrying # the actual message. @@ -1744,8 +1752,12 @@ def _trampoline(_ctx, request_ptr, response_ptr): # so returning non-zero without setting it could otherwise # surface a stale error. try: + # Returned message size limited here + text = str(e).replace("\x00", "\\x00") + if len(text) > 1024: + text = text[:1024] + "...(truncated)" _lib.c2pa_error_set_last( - "Other: Python HTTP resolver failed: {}".format(e) + f"Other: HTTP resolver trampoline failure: {text}" .encode("utf-8", "replace")) except BaseException: pass diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index bc6f6c69..a2e5a798 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -26,6 +26,10 @@ The resolver is validated immediately: something with neither shape raises `Type [`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class (subclassing it is not required, it just gets you a documented, type-checkable contract instead of duck typing), plus three example resolvers: `DebugHttpResolver` (logs every request/response, delegates the transfer to `urllib`), `CachingHttpResolver` (TTL'd LRU cache plus retry/backoff for throttled requests), and `AlwaysFailResolver` (answers every request with a fixed status; needs no network). The module has no dependency on `c2pa` itself; only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. +## Notes on runtime + +The native library and the Python process must share a C runtime: buffers the HTTP resolver bridge allocates are freed on the native side with `libc::free`, so a native library built against a different C runtime (e.g. a static-musl `.so`) than the one your Python process loads will corrupt memory. + ## What traffic flows through a resolver (and when) A resolver attached to a `Context` handles **every** HTTP request the SDK makes through that `Context`. Scope and gating to keep in mind: @@ -64,13 +68,14 @@ Conceptually the pipeline is: native code decides it needs an HTTP resource, cal - **The resolver outlives `Context.close()`.** Native `Reader`/`Builder` instances hold their own reference to the underlying native context, so your resolver can still be invoked after the Python `Context` is closed, for as long as any `Reader` or `Builder` created from it is alive. Do not tear down resolver resources (close a session, release a pool) on `Context.close()`; tie them to the resolvers' own lifetime instead. The SDK internally pins the callback thunk to keep this safe, so there is nothing you need to hold onto yourself. - **No reentrancy.** Do not call c2pa APIs from inside `resolve()`. Re-entering the FFI while a call is in flight is undefined. +- **Concurrent close() is unsafe.** Do not close a `Context`/`Reader`/`Builder` from one thread while another thread still has a call in flight on that same object, including a resolver call it triggered. This is a general property of these objects, not something specific to resolvers, but resolver users are the most likely to be multi-threaded. ## What the SDK does and does not do with HTTP The SDK delegates the *transfer* entirely; the resolver *is* the HTTP client. That means: - **No redirects.** The SDK does not follow redirects; a `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as both examples do) gives you redirect handling for free. A hand-rolled resolver must implement it. -- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; if you need an allowlist, enforce it yourself in `resolve()` (raise or return an error status for disallowed hosts). +- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; if you need an allowlist, enforce it yourself in `resolve()` (raise or return an error status for disallowed hosts). The URL is attacker-influenced: it comes from a remote-manifest reference embedded in whatever asset someone hands the application, so a resolver that hands it straight to a general-purpose HTTP client without a scheme check is a `file://` local-file-read and internal-network SSRF gadget. Both example resolvers reject any URL whose scheme is not `http`/`https` before doing anything else, for this reason. - **TLS is yours.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack your resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives; see the platform section below. - **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. If you serve large manifests, be aware the failure mode downstream is a validation error, not a size error. - **No caching, retries, or backoff.** Each needed resource is requested; policy is yours. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. diff --git a/tests/http_resolver/http_resolver_example_impl.py b/tests/http_resolver/http_resolver_example_impl.py index 05c6a4f2..9098a68d 100644 --- a/tests/http_resolver/http_resolver_example_impl.py +++ b/tests/http_resolver/http_resolver_example_impl.py @@ -26,10 +26,20 @@ import threading import time import urllib.error +import urllib.parse import urllib.request from abc import ABC, abstractmethod +def _reject_non_http(url): + """Raise if url's scheme is not http/https. + """ + scheme = urllib.parse.urlsplit(url).scheme.lower() + if scheme not in ("http", "https"): + raise ValueError( + f"refusing to resolve non-http(s) URL scheme: {scheme!r}") + + class HttpRequest: """An HTTP request the SDK asks a custom resolver to perform. @@ -155,6 +165,7 @@ def __init__(self, cache=None, timeout=10.0, max_retries=3, self._max_retry_after = float(max_retry_after) def resolve(self, request): + _reject_non_http(request.url) cacheable = request.method.upper() == "GET" if cacheable: cached = self.cache.get(request.url) @@ -218,6 +229,7 @@ def __init__(self, timeout=10.0): self.requests = [] def resolve(self, request): + _reject_non_http(request.url) self.requests.append((request.method, request.url)) # Timestamp requests POST a body; manifest fetches send none. From ad7d51c05f2fd96ed145026d08d6d20d29fc6afd Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:35:42 -0700 Subject: [PATCH 56/67] fix: Fix format --- src/c2pa/c2pa.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 95f00b5c..19edc3de 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1643,7 +1643,6 @@ def _parse_header_lines(raw: str) -> dict: headers[name.strip()] = value.strip() return headers - @staticmethod def _coerce_resolver(resolver): """Normalize a HTTP resolver into a callable taking an HttpRequest. From 6d6e4e68cdfa0e4c5bb1201ad57fe67e4844fa12 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:56:36 -0700 Subject: [PATCH 57/67] fix: Docs clean up --- pr303-adversarial-review-memory-callbacks.md | 97 -------------------- pr303-rereview-after-hardening.md | 54 +++++++++++ tests/http_resolver/README.md | 8 +- 3 files changed, 59 insertions(+), 100 deletions(-) delete mode 100644 pr303-adversarial-review-memory-callbacks.md create mode 100644 pr303-rereview-after-hardening.md diff --git a/pr303-adversarial-review-memory-callbacks.md b/pr303-adversarial-review-memory-callbacks.md deleted file mode 100644 index 8e699ce9..00000000 --- a/pr303-adversarial-review-memory-callbacks.md +++ /dev/null @@ -1,97 +0,0 @@ -# PR #303 — adversarial review: memory, callback, and resolver handling - -**Branch state reviewed:** `mathern/http-resolver-custom` at `c785450`. -**Method:** every claim below was adversarially confirmed — against the c2pa-rs FFI source at the pinned tag `c2pa-v0.90.1` (`c2pa_c_ffi/src/c_api.rs`), and where ctypes semantics decide the outcome, by running the experiment (results quoted). The exposed API is untouched by this plan, per instruction. Implementer: Opus 5. - ---- - -## Part A — Confirmed sound. Do not "fix" these. - -Each of these was attacked and held. They are listed so the implementer doesn't churn them. - -**A1. The bytearray snapshot fix (`data = bytes(payload)`) is complete and correct.** Before `184790f`, `length = len(payload)` was computed on the live object and `bytes(payload)` was taken later at `memmove` time; a resolver returning a `bytearray` and shrinking it from another thread in between would have made `memmove` read `length` bytes from a shorter source — an out-of-bounds read. The current code snapshots once and derives `length` from the snapshot. The parallel fix in the signer callback (`signature = bytes(signature)`) closes the same window there. Verified: `bytes(bytearray)` produces an immutable copy. - -**A2. The zero-length body invariant matches Rust exactly.** On the success path Rust skips its `free` when `body.is_null() || body_len == 0`, so a non-NULL pointer with zero length would leak; the trampoline writes `body = None` / `body_len = 0` together. On the error path (`rc != 0`) Rust frees any non-null body unconditionally — and on the trampoline's only mid-write failure (malloc returning NULL), `status` may already be written but `body` is still NULL from Rust's zero-init (`C2paHttpResponse { status: 0, body: null, body_len: 0 }` is constructed fresh per call), so nothing dangles and nothing leaks. - -**A3. The allocator-matching logic is right for every shipped artifact.** Rust frees with `libc::free`; `ucrtbase`-before-`msvcrt` matches Rust MSVC targets, and `CDLL(None)` matches on Unix. Checked `build-wheel.yml`: wheels are `manylinux_2_28` x86_64/aarch64 (glibc) plus mac/Windows — no musllinux target, so there is no shipped configuration where `CDLL(None)` and the native library disagree (see B8 for the residual source-build note). The lazy-init race on `_native_malloc` is benign (idempotent assignment of equivalent objects). - -**A4. The thunk keep-alive chain holds, including failure paths.** `_http_resolver_cb` is pinned on the Context before any native call can capture the raw pointer; `Context._release` deliberately does not clear it; `Reader`/`Builder` store `_context` on construction and clear it only in their own release, at which point their native Arc clone is already being dropped. If `Context.__init__` fails after the resolver was set (signer rejection, build failure), the `with self._NativeBuilder()` block frees the native builder, which owns the resolver by then (`set_http_resolver` does `Box::from_raw` — verified), so no native user of the thunk survives the exception. `_NativeHttpResolver` failure before consumption is freed by its own `with` block. No leak, no dangle, in any ordering I could construct under the documented single-owner usage rules. - -**A5. No uncollectable cycles, no error-slot cross-thread confusion.** The thunk's closure chain (`Context → CFUNCTYPE → _trampoline → resolve_fn → resolver`) has no back-edge to Context unless the user creates one, and PEP 442 collects cycles through `__del__` anyway. `c2pa_error_set_last` writes a thread-local slot and the callback is synchronous, so the error is always set on exactly the thread that reads it after `rc != 0`. - -**A6. Header parsing is injection-safe.** Rust builds the block from `http::HeaderMap` (names already lowercase, values cannot contain `\n` — `to_str()` filters), joins with `\n`, and always sends a valid, possibly empty, never-NULL CString. `partition(":")` keeps colons inside values. The Python side's NULL-guard is defensively redundant, which is fine. - ---- - -## Part B — Findings. Ordered by severity; each with its adversarial confirmation and the fix Opus 5 should implement. - -### B1 (High, security) — The reference resolvers are SSRF/local-file-read gadgets - -The request URL a resolver receives originates from **untrusted asset content**: a remote-manifest reference embedded in whatever JPEG someone hands the application. Both `DebugHttpResolver` and `CachingHttpResolver` pass `request.url` straight to `urllib.request.urlopen`, whose default opener chain includes `FileHandler`, `FTPHandler`, and `DataHandler`. Confirmed consequences: - -- A crafted asset with a `file:///etc/hostname`-style manifest URL makes the resolver read a **local file** and feed its contents into the SDK (and, for `CachingHttpResolver`, into the cache; for `DebugHttpResolver`, the URL into its log). -- `http://169.254.169.254/...` or any internal-network URL is fetched **from inside the caller's network** — the classic SSRF the built-in resolver's `core.allowed_network_hosts` exists to prevent, and which a custom resolver explicitly bypasses. -- `data:` URLs let the asset author synthesize the "fetched" bytes with no network at all. - -The built-in Rust resolver speaks only http/https; the examples silently widen that. Since this directory is the copy-paste reference implementation, the examples are the spec — they must model the safe pattern. - -**Fix:** in `http_resolver_example_impl.py`, both network-delegating resolvers reject non-`http(s)` schemes up front (parse with `urllib.parse.urlsplit`; on any other scheme, raise — a hard failure is the right semantic for "this URL should never be fetched"). Add a short "the URL is attacker-influenced" paragraph to `tests/http_resolver/README.md`'s host-filtering bullet, naming `file://` and internal-host SSRF explicitly. Add a no-network test: build a request-shaped object with a `file://` URL, call `resolve()` directly, assert it raises without touching the filesystem. - -### B2 (Medium, data integrity) — Response status is silently double-truncated; a non-200 can alias to 200 - -Two modular reductions sit between the resolver's return value and what validation sees. Confirmed empirically on the Python side: assigning `2**35 + 200` to a `c_int` struct field **does not raise — it stores 200**. Confirmed in Rust source: `Response::builder().status(resp.status as u16)` truncates i32 → u16. Net effect: status `65736` (or `2**32 + 200`, etc.) reaches validation as a clean `200`; a negative status wraps. A malicious resolver gains nothing (it could return 200 honestly), so this is not an escalation — but a *buggy* resolver (an off-by-arithmetic on a retry counter mixed into status, a status parsed from a corrupt upstream) has its error laundered into success, which for a remote-manifest fetch means "manifest accepted" instead of "fetch failed". Silent modular arithmetic in a validation pipeline is corruption by another name. - -**Fix:** in the trampoline, after the existing int/bool check: `if not 100 <= status <= 599: raise TypeError(f"resolver response .status out of range: {status}")`. The existing `BaseException` handler converts it into a typed error carrying that message. Test: resolver returning `65736` produces a `C2paError` mentioning the out-of-range value, not a success. - -### B3 (Medium, contract integrity) — Falsy non-bytes bodies are silently masked to empty - -`payload = result.body or b""` runs **before** the isinstance check, so `result.body = 0`, `False`, `0.0`, or `[]` all become `b""` and pass — confirmed: `0 or b""` → `b""`. The check that exists specifically to catch wrong-typed bodies has a falsy-shaped hole in it; a resolver bug returning `0` (say, a misplaced status) yields a well-formed empty 200 response instead of a diagnostic. Same family as B2: an error becomes silently valid data. - -**Fix:** replace with explicit None-handling: `payload = result.body; if payload is None: payload = b""` then the existing isinstance check (which now sees `0` and raises). Test: resolver whose response has `body = 0` produces a typed error naming `int`, and `body = None` still means empty body. - -### B4 (Medium, correctness/robustness) — The signer callback lags the trampoline's hardening on three counts - -The resolver trampoline and the signer callback are the two Python→native callbacks in the file; the review compared them line by line. The signer's `wrapped_callback`: - -1. **Catches `Exception`, not `BaseException`.** A `KeyboardInterrupt`/`SystemExit` raised inside the user's signing callback is swallowed by ctypes' own callback guard (ctypes never unwinds into native code — it reports the exception and returns the restype default, `0`). `0` is neither the `-1` error convention nor a plausible signature length; whether native code treats "0 bytes signed" as failure is left to chance, and the real cause is lost. The trampoline's `BaseException` catch exists for exactly this; mirror it. -2. **Never calls `c2pa_error_set_last`.** By the bridge's own (verified) analysis, the native error slot is thread-local and *not cleared* before callbacks — returning `-1` without setting it can surface a **stale error from an earlier call** as the signing failure's diagnosis. The trampoline sets the slot on every failure path; the signer sets it on none. -3. **Silently truncates oversized signatures.** `actual_len = min(len(signature), signed_len)` — a signature longer than the native buffer is cut and returned as if complete, guaranteeing a baffling downstream validation failure instead of a clear local error. Truncated cryptographic material is corruption, not accommodation. - -**Fix:** catch `BaseException`; on every failure path call `c2pa_error_set_last` with a `"Other: Python signer callback failed: ..."` message (same sanitization as B5) before returning `-1`; and replace the `min()` with `if len(signature) > signed_len: set error slot; return -1`. Keep the existing `-1` returns for the input-validation branches but add slot messages there too. Test (unit-level, no signing round-trip needed): a callback returning an oversized `bytes` yields a typed error, not a truncated signature. - -### B5 (Low, diagnostics) — Error-slot messages are NUL-truncatable and unbounded - -`"Other: Python HTTP resolver failed: {}".format(e).encode("utf-8", "replace")` — confirmed empirically that a `bytes` argument with an embedded NUL passes through `c_char_p` fine and the C side simply stops at the NUL. Exception text is partly attacker-influenceable (server-supplied strings inside `URLError`/`HTTPError` messages), so a hostile upstream can blank out the diagnostic tail. Separately, nothing caps the size: a pathological exception message makes Rust build an equally pathological `CString`. Neither is memory-unsafe (that's the point of the confirmation); both degrade the one artifact you need when debugging a resolver in production. - -**Fix:** one tiny helper used by both callbacks (B4 makes the signer need it too): take `str(e)`, replace `"\x00"` with `"\\x00"`, truncate to ~1 KB with an ellipsis marker, then encode. The `"Other: "` prefix is already fixed-position, so error-type spoofing via a crafted message (`"Signature: ..."`) is not possible — confirmed against Rust's first-colon parse; keep the prefix exactly as is. - -### B6 (Low, hardening) — The `rc=0`-with-partial-response hazard is one careless edit away - -The trampoline's safety story depends on a structural invariant nobody wrote down: **no fallible statement may sit between the first write into the response struct and `return 0`.** Today that's true (status write → body write → return). If a future edit inserts anything that can raise after `response.status = status`, the `BaseException` handler returns `-1` and Rust's error path cleans up correctly (frees a body if one was written) — that part is safe. The nastier variant is an edit that makes the *ctypes guard itself* the returner (an exception the handler can't run for, e.g. during interpreter teardown): ctypes returns the restype default `0`, and Rust reads a half-written struct as success. Can't be triggered today; cheap to fence off forever. - -**Fix:** a loud comment at the write site stating the invariant, and — the actually load-bearing part — reorder so `response.status` is written **last**, after the body fields. Then any hypothetical partial state is "body set, status still 0", which Rust's `Response::builder().status(0)` rejects as an invalid status code rather than accepting. Zero-cost, converts the worst theoretical outcome from silent-success to clean failure. - -### B7 (Low, optional hardening) — Reentrancy is documented UB; it could be a clean error instead - -"Do not call c2pa APIs from inside the resolver" is currently enforced by hoping. A thread-local `_in_native_callback` flag set around the `resolve_fn` invocation (and the signer callback body), checked by `_ensure_valid_state` / the few module-level entry points, converts a re-entrant call into an immediate `C2paError("c2pa APIs must not be called from inside a resolver/signer callback")` on the exact offending line. Cost: one thread-local read on the hot path. Recommended, but genuinely optional — decline it if the entry-point audit gets invasive. - -### B8 (Notes; docstring-only, no renames — API surface stays as instructed) - -- The ctypes structs `C2paHttpRequest` / `C2paHttpResponse` / `C2paHttpResolver` carry docstrings saying they're "useful if planning to write custom HTTP resolvers." They aren't — resolver authors never touch ctypes; these are trampoline internals. Since the exposed API must not change, fix only the docstrings (e.g. "Internal ctypes mirror of the native struct; resolver authors interact with the decoded `C2paHttpRequestData` instead"). -- Source builds on musl (Alpine) are the one configuration where `CDLL(None)` could disagree with the native library's allocator if someone produces a static-musl `.so`. Not reachable via shipped wheels (A3); one sentence in the build docs ("native artifact and Python must share a C runtime") closes it. -- Concurrent `close()` while another thread is mid-operation on the same object is a pre-existing, class-wide use-after-free hazard of `ManagedResource` (no in-flight guard exists), not introduced by this PR; the resolver merely adds the thunk to what dangles. Out of scope here — but the resolver docs' thread-safety note should say "do not close objects that another thread is still using" explicitly, since resolver users are the most likely to be multi-threaded. - ---- - -## Part C — Test additions (all no-network, deterministic) - -1. B1: `file://` URL → both example resolvers raise; nothing read from disk (assert via a path that would exist). -2. B2: status `65736` and status `-1` → typed `C2paError`, message names the value. -3. B3: body `0` → typed error naming `int`; body `None` → succeeds as empty. -4. B4: signer callback returning oversized signature → typed error, no truncation; signer callback raising `KeyboardInterrupt` → `-1` path with slot message (invoke `wrapped_callback` directly with ctypes buffers). -5. Regression for A1: resolver returns a `bytearray` and mutates it from a timer thread immediately after `resolve()` returns; signed/validated result must be built from the snapshot (deterministic version: mutate in a subclass's `__buffer__`-free wrapper before returning is enough to prove the copy point — keep it simple: assert the trampoline's copy equals the pre-mutation content by round-tripping through `AlwaysFailResolver`-style in-memory serving). -6. B5: resolver raising `Exception("boom\x00hidden")` → resulting `C2paError` message contains `boom` and the escaped marker, not a truncation at the NUL. - -## Part D — Implementation order - -B1 (examples + README + test) → B2/B3 together (same function, same test file) → B5 helper → B4 signer (uses the helper) → B6 reorder+comment → B7 if clean → B8 docstrings. Each step is independently landable; nothing here touches the exposed API, the FFI signatures, or the lifecycle machinery verified in Part A. diff --git a/pr303-rereview-after-hardening.md b/pr303-rereview-after-hardening.md new file mode 100644 index 00000000..9bd02622 --- /dev/null +++ b/pr303-rereview-after-hardening.md @@ -0,0 +1,54 @@ +# PR #303 — re-review after the hardening pass + +**Branch state:** `mathern/http-resolver-custom` at `377cd27` ("fix: hardening"). +**Baseline:** the prior adversarial review at `c785450`. +**Method:** every accepted change was re-derived against the c2pa-rs FFI at the pinned tag `c2pa-v0.90.1` and, where ctypes/urllib semantics decide it, re-run (results quoted). Signer callback confirmed untouched, as you noted. + +## Bottom line + +The four accepted fixes are all **correctly implemented and adversarially confirmed sound** — including two subtleties the fixes could have gotten wrong but didn't. There is **one new, CI-breaking regression** introduced by the refactor (a lint error), and a few small notes. Nothing here is memory-unsafe. The one must-fix is trivial. + +--- + +## Part 1 — Accepted fixes: re-verified + +**B2 (status range check) — correct, and tight.** `if not 100 <= status <= 599` now sits after the int/bool check. Confirmed the double-truncation is fully closed: 599 is the max accepted, far below both i32 and u16 limits, so there is no value that passes the Python check and then changes meaning under Rust's `status as u16`. Boundary-tested: 100/599 accepted; 99/600/65736/-1 rejected. The one behavioral consequence worth a mention: nonstandard 6xx/7xx statuses that some CDNs emit now raise instead of passing through. That is a defensible trade (it's the price of killing the truncation alias) — just flag it in the `with_resolver` notes so it isn't a surprise. + +**B3 (falsy-body mask) — correct.** `payload = result.body; if payload is None: payload = b""` replaced the `or b""`. Re-confirmed the hole is closed: a `0` / `False` / `[]` body now reaches the isinstance check and raises `TypeError` naming the type, while `None` still legitimately means empty. The `data = bytes(payload)` snapshot (the A1 fix from last round) remains correctly placed right after the type check. + +**B5 (error-message hardening) — correct, and safe in its own edge cases.** NUL is escaped (`replace("\x00", "\\x00")`), length capped at 1024 codepoints + `"...(truncated)"`, then `encode(..., "replace")`. Confirmed: the truncation slices on codepoints (not bytes), so no mid-character mojibake; the resulting CString is bounded. Also checked the failure mode of the size-limiting block itself — if a pathological `__str__` raises, the surrounding `except BaseException: pass` swallows it and the slot goes unset (degrades to the "stale slot" case, not a crash). Acceptable. The `"Other: "` prefix is preserved, so error-type spoofing via a crafted message remains impossible. + +**B6 (write-status-last) — correct, and its safety claim actually holds against Rust.** This is the one I attacked hardest, because the comment makes a specific promise. Verified end to end: +- The body fields are written first; `response.status = status` is the last statement before `return 0`. +- The promised property is "a partial write leaves `status = 0`, which Rust rejects." Confirmed on both sides: Rust constructs the response struct fresh per call as `{status: 0, body: null, body_len: 0}`, and `Response::builder().status(0 as u16)` fails (`StatusCode::from_u16(0)` is an error), so a half-written struct read as success by the ctypes guard's default `0` return becomes a **clean Rust-side failure**, not accepted data. +- The subtler question — does the malloc'd body leak in that partial-write case? — is **no**. Rust's `TryFrom` copies and `libc::free`s the body *before* it builds the Response and hits the invalid-status error (verified in `c_api.rs`: the free happens in the body-extraction block that precedes `.status().body()`). So even the partial-success path frees correctly. The reorder is strictly a safety improvement with zero cost. + +**B1 (SSRF/local-file — example resolvers) — correct, no bypass found.** `_reject_non_http` uses `urllib.parse.urlsplit(url).scheme.lower()` and both example resolvers call it as their first line. Attacked the scheme check directly: +- `file://`, `FILE://`, `ftp://`, `gopher://`, `data:`, `jar:file://`, and scheme-relative `//evil` are all **blocked**. Good. +- Probed for check-vs-client divergence (a URL the check reads as http but `urlopen` fetches as something else, or vice-versa). The only divergences found — embedded-newline schemes like `"http\n://evil"` — **fail closed**: the check may pass them, but `urllib.request.urlopen` then raises `URLError: unknown url type` and performs no fetch. Confirmed by running it. So there is no "allowed by check, fetched anyway" path. The fix is sound. +- The README bullet correctly frames the URL as attacker-influenced and names both `file://` read and internal-network SSRF. + +**B8 docstrings + docs — done and accurate.** The ctypes struct docstrings no longer claim to be a resolver-author surface; the musl/C-runtime note and the concurrent-`close()` warning both landed in the README with correct framing. + +--- + +## Part 3 — Rejected items: my read on the rejections + +You said some changes were rejected; reconciling against what shipped: + +- **B4 (signer-callback hardening) — rejected, and you confirmed no signer changes.** For the record so it's a deliberate decision and not a silent gap: the signer callback still catches `Exception` (not `BaseException`), still never calls `c2pa_error_set_last`, and still truncates an oversized signature via `min(len, signed_len)`. None of these is memory-unsafe — they're diagnostic/robustness gaps, and the signing path is out of this PR's stated scope (HTTP resolver). Leaving it is a reasonable scoping call. If you ever want it, it's a self-contained follow-up; no dependency on anything here. +- **B7 (reentrancy guard) — rejected.** Fine. It was explicitly marked optional last round. Reentrancy stays documented-UB rather than a caught error. No residue in the code from a partial attempt — clean rejection. + +Neither rejection leaves a correctness or safety hole in the resolver path. + +--- + +## Part 4 — Smaller notes (optional) + +- **N1 (test coverage gap).** The three new guards (B1 scheme reject, B2 range, B3 falsy body) and the B5 message shaping have **no tests**. The guards are correct, but they're exactly the kind of thing a future refactor silently removes. Cheap, all no-network: `file://` URL → both example resolvers raise (assert nothing read from disk); status `65736`/`-1` → typed error naming the value; body `0` → typed error naming `int`, body `None` → empty ok; exception message `"boom\x00hidden"` → resulting `C2paError` contains `boom` and the escaped marker. These are the same six from the prior plan's Part C, minus the signer ones. Worth adding before merge since this directory is the reference. +- **N2 (status upper bound).** 599 excludes the rare nonstandard 6xx/7xx. If any target environment legitimately sees those, widen to `<= 999` (still within u16, still kills the truncation alias). Default to 599 unless you know otherwise — just don't let it surprise anyone. +- **N3.** Two pre-existing E501s in `http_resolver_example_impl.py` (lines 18, 55) are untouched by this PR and not linted by CI (only `src/c2pa/c2pa.py` is checked), so they're harmless — noting them only so they're not mistaken for new. + +## Fix order + +R1 (delete the blank line — unblocks CI) → N1 tests if you want them in this PR → N2 only if your environments need it. Everything in Part 1 is done and verified; don't touch it. diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index a2e5a798..c2c40a2f 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -8,7 +8,7 @@ A custom HTTP resolver lets you intercept every HTTP request the SDK makes throu c2pa ships no resolver types at all: there is no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `ContextBuilder.with_resolver()` (and the `Context(resolver=...)` constructor argument, also accepted by `Context.from_json`/`from_dict`) never does an `isinstance` check. It accepts either: -- any callable taking one request argument, or +- any callable taking one request argument; - any object with a `resolve(request)` method. The request handed to you exposes `.url` (str), `.method` (str), `.headers` (dict), and `.body` (bytes). The value you return only needs `.status` (int) and `.body` (bytes) attributes. The minimal form needs no imports at all: @@ -22,7 +22,7 @@ context = c2pa.Context.builder().with_resolver(my_resolver).build() reader = c2pa.Reader("image/jpeg", stream, context=context) ``` -The resolver is validated immediately: something with neither shape raises `TypeError` from `with_resolver()` itself (or from the `Context` constructor on the direct path), not later at `.build()`. If a resolver object has both a `resolve()` method and is itself callable, `resolve()` wins. +The resolver is validated immediately: something with a wrong shape raises `TypeError` from `with_resolver()` itself (or from the `Context` constructor on the direct path), not later at `.build()`. If a resolver object has both a `resolve()` method and is itself callable, `resolve()` wins. [`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class (subclassing it is not required, it just gets you a documented, type-checkable contract instead of duck typing), plus three example resolvers: `DebugHttpResolver` (logs every request/response, delegates the transfer to `urllib`), `CachingHttpResolver` (TTL'd LRU cache plus retry/backoff for throttled requests), and `AlwaysFailResolver` (answers every request with a fixed status; needs no network). The module has no dependency on `c2pa` itself; only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. @@ -114,7 +114,9 @@ Default `multiprocessing` start methods differ: historically `fork` on Linux (`f - [`test_http_resolver_debug.py`](./test_http_resolver_debug.py): exercises `DebugHttpResolver`, verifying that a remote-manifest read logs a `GET`, that signing with a remote-manifest ingredient fetches it through the resolver, and that re-reading the signed (embedded-manifest) output performs no HTTP at all. - [`test_http_resolver_cache.py`](./test_http_resolver_cache.py): exercises `CachingHttpResolver` (reading twice / ingesting the same ingredient repeatedly hits the cache exactly as the hit/miss counters predict) and `AlwaysFailResolver` (a non-200 answer surfaces as a clean typed `C2paError`, with no network needed). -Both network tests need internet access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above); on macOS, prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. +Those examples need network access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. + +If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above); on macOS, prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. Run them in a CLI with the commands: From 701d716a7cd2dec07ec6f9c3e11697f9c40bebb7 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:56:49 -0700 Subject: [PATCH 58/67] fix: Docs clean up 2 --- pr303-rereview-after-hardening.md | 54 ------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 pr303-rereview-after-hardening.md diff --git a/pr303-rereview-after-hardening.md b/pr303-rereview-after-hardening.md deleted file mode 100644 index 9bd02622..00000000 --- a/pr303-rereview-after-hardening.md +++ /dev/null @@ -1,54 +0,0 @@ -# PR #303 — re-review after the hardening pass - -**Branch state:** `mathern/http-resolver-custom` at `377cd27` ("fix: hardening"). -**Baseline:** the prior adversarial review at `c785450`. -**Method:** every accepted change was re-derived against the c2pa-rs FFI at the pinned tag `c2pa-v0.90.1` and, where ctypes/urllib semantics decide it, re-run (results quoted). Signer callback confirmed untouched, as you noted. - -## Bottom line - -The four accepted fixes are all **correctly implemented and adversarially confirmed sound** — including two subtleties the fixes could have gotten wrong but didn't. There is **one new, CI-breaking regression** introduced by the refactor (a lint error), and a few small notes. Nothing here is memory-unsafe. The one must-fix is trivial. - ---- - -## Part 1 — Accepted fixes: re-verified - -**B2 (status range check) — correct, and tight.** `if not 100 <= status <= 599` now sits after the int/bool check. Confirmed the double-truncation is fully closed: 599 is the max accepted, far below both i32 and u16 limits, so there is no value that passes the Python check and then changes meaning under Rust's `status as u16`. Boundary-tested: 100/599 accepted; 99/600/65736/-1 rejected. The one behavioral consequence worth a mention: nonstandard 6xx/7xx statuses that some CDNs emit now raise instead of passing through. That is a defensible trade (it's the price of killing the truncation alias) — just flag it in the `with_resolver` notes so it isn't a surprise. - -**B3 (falsy-body mask) — correct.** `payload = result.body; if payload is None: payload = b""` replaced the `or b""`. Re-confirmed the hole is closed: a `0` / `False` / `[]` body now reaches the isinstance check and raises `TypeError` naming the type, while `None` still legitimately means empty. The `data = bytes(payload)` snapshot (the A1 fix from last round) remains correctly placed right after the type check. - -**B5 (error-message hardening) — correct, and safe in its own edge cases.** NUL is escaped (`replace("\x00", "\\x00")`), length capped at 1024 codepoints + `"...(truncated)"`, then `encode(..., "replace")`. Confirmed: the truncation slices on codepoints (not bytes), so no mid-character mojibake; the resulting CString is bounded. Also checked the failure mode of the size-limiting block itself — if a pathological `__str__` raises, the surrounding `except BaseException: pass` swallows it and the slot goes unset (degrades to the "stale slot" case, not a crash). Acceptable. The `"Other: "` prefix is preserved, so error-type spoofing via a crafted message remains impossible. - -**B6 (write-status-last) — correct, and its safety claim actually holds against Rust.** This is the one I attacked hardest, because the comment makes a specific promise. Verified end to end: -- The body fields are written first; `response.status = status` is the last statement before `return 0`. -- The promised property is "a partial write leaves `status = 0`, which Rust rejects." Confirmed on both sides: Rust constructs the response struct fresh per call as `{status: 0, body: null, body_len: 0}`, and `Response::builder().status(0 as u16)` fails (`StatusCode::from_u16(0)` is an error), so a half-written struct read as success by the ctypes guard's default `0` return becomes a **clean Rust-side failure**, not accepted data. -- The subtler question — does the malloc'd body leak in that partial-write case? — is **no**. Rust's `TryFrom` copies and `libc::free`s the body *before* it builds the Response and hits the invalid-status error (verified in `c_api.rs`: the free happens in the body-extraction block that precedes `.status().body()`). So even the partial-success path frees correctly. The reorder is strictly a safety improvement with zero cost. - -**B1 (SSRF/local-file — example resolvers) — correct, no bypass found.** `_reject_non_http` uses `urllib.parse.urlsplit(url).scheme.lower()` and both example resolvers call it as their first line. Attacked the scheme check directly: -- `file://`, `FILE://`, `ftp://`, `gopher://`, `data:`, `jar:file://`, and scheme-relative `//evil` are all **blocked**. Good. -- Probed for check-vs-client divergence (a URL the check reads as http but `urlopen` fetches as something else, or vice-versa). The only divergences found — embedded-newline schemes like `"http\n://evil"` — **fail closed**: the check may pass them, but `urllib.request.urlopen` then raises `URLError: unknown url type` and performs no fetch. Confirmed by running it. So there is no "allowed by check, fetched anyway" path. The fix is sound. -- The README bullet correctly frames the URL as attacker-influenced and names both `file://` read and internal-network SSRF. - -**B8 docstrings + docs — done and accurate.** The ctypes struct docstrings no longer claim to be a resolver-author surface; the musl/C-runtime note and the concurrent-`close()` warning both landed in the README with correct framing. - ---- - -## Part 3 — Rejected items: my read on the rejections - -You said some changes were rejected; reconciling against what shipped: - -- **B4 (signer-callback hardening) — rejected, and you confirmed no signer changes.** For the record so it's a deliberate decision and not a silent gap: the signer callback still catches `Exception` (not `BaseException`), still never calls `c2pa_error_set_last`, and still truncates an oversized signature via `min(len, signed_len)`. None of these is memory-unsafe — they're diagnostic/robustness gaps, and the signing path is out of this PR's stated scope (HTTP resolver). Leaving it is a reasonable scoping call. If you ever want it, it's a self-contained follow-up; no dependency on anything here. -- **B7 (reentrancy guard) — rejected.** Fine. It was explicitly marked optional last round. Reentrancy stays documented-UB rather than a caught error. No residue in the code from a partial attempt — clean rejection. - -Neither rejection leaves a correctness or safety hole in the resolver path. - ---- - -## Part 4 — Smaller notes (optional) - -- **N1 (test coverage gap).** The three new guards (B1 scheme reject, B2 range, B3 falsy body) and the B5 message shaping have **no tests**. The guards are correct, but they're exactly the kind of thing a future refactor silently removes. Cheap, all no-network: `file://` URL → both example resolvers raise (assert nothing read from disk); status `65736`/`-1` → typed error naming the value; body `0` → typed error naming `int`, body `None` → empty ok; exception message `"boom\x00hidden"` → resulting `C2paError` contains `boom` and the escaped marker. These are the same six from the prior plan's Part C, minus the signer ones. Worth adding before merge since this directory is the reference. -- **N2 (status upper bound).** 599 excludes the rare nonstandard 6xx/7xx. If any target environment legitimately sees those, widen to `<= 999` (still within u16, still kills the truncation alias). Default to 599 unless you know otherwise — just don't let it surprise anyone. -- **N3.** Two pre-existing E501s in `http_resolver_example_impl.py` (lines 18, 55) are untouched by this PR and not linted by CI (only `src/c2pa/c2pa.py` is checked), so they're harmless — noting them only so they're not mistaken for new. - -## Fix order - -R1 (delete the blank line — unblocks CI) → N1 tests if you want them in this PR → N2 only if your environments need it. Everything in Part 1 is done and verified; don't touch it. From 0cd2a252d4a719baba94310432819b63686605b5 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:39:32 -0700 Subject: [PATCH 59/67] fix: Docs clean up 3 --- tests/http_resolver/README.md | 12 +++++++++++- .../http_resolver/test_http_resolver_cache.py | 18 ++++++++++++++++-- .../http_resolver/test_http_resolver_debug.py | 15 +++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index c2c40a2f..717bfb40 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -116,7 +116,7 @@ Default `multiprocessing` start methods differ: historically `fork` on Linux (`f Those examples need network access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. -If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above); on macOS, prefix the commands with `SSL_CERT_FILE=$(python -m certifi)`. +If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above); on macOS python.org builds this is the common case, so the commands below include the fix. Run them in a CLI with the commands: @@ -127,3 +127,13 @@ python ./tests/http_resolver/test_http_resolver_cache.py # Or the whole directory via unittest discovery: python -m unittest discover -s tests/http_resolver -v ``` + +On some operating systems, if you hit CERTIFICATE_VERIFY_FAILED, this prefix on the commands fixes it: + +```bash +SSL_CERT_FILE=$(python -m certifi) python ./tests/http_resolver/test_http_resolver_debug.py +SSL_CERT_FILE=$(python -m certifi) python ./tests/http_resolver/test_http_resolver_cache.py + +# Or the whole directory via unittest discovery: +SSL_CERT_FILE=$(python -m certifi) python -m unittest discover -s tests/http_resolver -v +``` diff --git a/tests/http_resolver/test_http_resolver_cache.py b/tests/http_resolver/test_http_resolver_cache.py index f7d4c0ed..443aed07 100644 --- a/tests/http_resolver/test_http_resolver_cache.py +++ b/tests/http_resolver/test_http_resolver_cache.py @@ -49,15 +49,20 @@ def test_read_with_cache(self): If the SDK ever adds another GET here, this count shifts -- that's expected and worth updating, not a false alarm. """ + print('\n--- Reading using an HTTP resolver with a cache') resolver = CachingHttpResolver() with c2pa.Context.builder().with_resolver( resolver).build() as context: - for _ in range(2): + for i in range(2): + print(f"Read #{i + 1} of cloud.jpg") with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: with c2pa.Reader( "image/jpeg", f, context=context) as reader: reader.get_validation_state() + print(f"Cache state after read #{i + 1}: " + f"hits={resolver.cache.hits} " + f"misses={resolver.cache.misses}") self.assertEqual(resolver.cache.hits, 1) self.assertEqual(resolver.cache.misses, 1) @@ -70,6 +75,7 @@ def test_sign_with_cache(self): context resolver, so three adds mean one network fetch and two cache hits. """ + print('\n--- Reading and signing using an HTTP resolver with a cache') with open(os.path.join(FIXTURES, "es256_certs.pem"), "rb") as f: certs = f.read() with open(os.path.join(FIXTURES, "es256_private.key"), "rb") as f: @@ -117,11 +123,15 @@ def test_sign_with_cache(self): "rb") as ingredient: for index in range(3): ingredient.seek(0) + print(f"Adding ingredient #{index + 1} of 3 " + f"({ingredient_labels[index]})") builder.add_ingredient( {"title": f"cloud.jpg #{index + 1}", "relationship": "componentOf", "label": ingredient_labels[index]}, "image/jpeg", ingredient) + print(f"Cache state: hits={resolver.cache.hits} " + f"misses={resolver.cache.misses}") with tempfile.TemporaryDirectory() as output_dir: output_path = os.path.join( @@ -129,6 +139,7 @@ def test_sign_with_cache(self): with open(os.path.join(FIXTURES, "A.jpg"), "rb") as source: with open(output_path, "wb") as dest: + print(f"Signing asset A.jpg -> {output_path}") builder.sign( c2pa.Signer.from_info(signer_info), "image/jpeg", source, dest) @@ -136,6 +147,9 @@ def test_sign_with_cache(self): finally: context.close() + print(f"Final cache state: hits={resolver.cache.hits} " + f"misses={resolver.cache.misses} " + f"(expected 1 miss, 2 hits, across 3 identical ingredients)") self.assertEqual(resolver.cache.misses, 1) self.assertEqual(resolver.cache.hits, 2) @@ -146,7 +160,7 @@ def test_failing_resolver_is_a_clean_error(self): """ with c2pa.Context.builder().with_resolver( AlwaysFailResolver(500)).build() as context: - with self.assertRaises(c2pa.C2paError): + with self.assertRaises(c2pa.C2paError) as ctx: with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: c2pa.Reader("image/jpeg", f, context=context) diff --git a/tests/http_resolver/test_http_resolver_debug.py b/tests/http_resolver/test_http_resolver_debug.py index 7bd89519..cdf770fa 100644 --- a/tests/http_resolver/test_http_resolver_debug.py +++ b/tests/http_resolver/test_http_resolver_debug.py @@ -45,7 +45,9 @@ class TestHttpResolverDebug(unittest.TestCase): def test_read_with_resolver(self): """Reading an asset whose manifest lives at a remote URL logs a GET for that fetch.""" + print('\n--- Reading using an HTTP resolver logging requests') resolver = DebugHttpResolver() + print("Reading using a Context with DebugHttpResolver as HTTP resolver") with c2pa.Context.builder().with_resolver( resolver).build() as context: with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: @@ -54,7 +56,10 @@ def test_read_with_resolver(self): reader.get_validation_state() self.assertFalse(reader.is_embedded()) self.assertTrue(reader.get_remote_url()) + print(f"Remote manifest url: {reader.get_remote_url()}") + print(f"Resolver saw {len(resolver.requests)} request(s): " + f"{resolver.requests}") self.assertTrue( any(method == "GET" for method, _ in resolver.requests)) @@ -62,6 +67,7 @@ def test_sign_with_resolver(self): """Signing an asset with a remote-manifest ingredient logs a GET for the ingredient's manifest, and reading the signed result back makes no HTTP requests at all (its manifest is embedded).""" + print('\n--- Reading and signing using an HTTP resolver logging requests') with open(os.path.join(FIXTURES, "es256_certs.pem"), "rb") as f: certs = f.read() with open(os.path.join(FIXTURES, "es256_private.key"), "rb") as f: @@ -102,12 +108,15 @@ def test_sign_with_resolver(self): try: builder = c2pa.Builder(manifest_definition, context=context) + print("Adding cloud.jpg as an ingredient...") with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as ingredient: builder.add_ingredient( {"title": "cloud.jpg", "relationship": "componentOf", "label": "cloud-ingredient"}, "image/jpeg", ingredient) + print(f"Resolver saw {len(resolver.requests)} request(s) " + f"after add_ingredient: {resolver.requests}") with tempfile.TemporaryDirectory() as output_dir: output_path = os.path.join( @@ -115,6 +124,7 @@ def test_sign_with_resolver(self): with open(os.path.join(FIXTURES, "A.jpg"), "rb") as source: with open(output_path, "wb") as dest: + print(f"Signing A.jpg -> {output_path}") builder.sign( c2pa.Signer.from_info(signer_info), "image/jpeg", source, dest) @@ -125,6 +135,8 @@ def test_sign_with_resolver(self): # Reading the signed file back uses its embedded # manifest, so this makes no HTTP requests at all. + print("Re-reading the signed output " + "(should trigger no HTTP requests)") with open(output_path, "rb") as f: with c2pa.Reader("image/jpeg", f) as reader: store = json.loads(reader.json()) @@ -132,6 +144,9 @@ def test_sign_with_resolver(self): store["active_manifest"]] self.assertTrue(manifest.get("ingredients")) + print(f"Resolver saw {len(resolver.requests)} request(s) " + f"total (expected {requests_before_reread}, " + "unchanged since re-read is embedded-only)") self.assertEqual( len(resolver.requests), requests_before_reread) finally: From 95dd713e95c4b68cc612569df79b2f243b068714 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:41:10 -0700 Subject: [PATCH 60/67] fix: Docs clean up once more --- tests/http_resolver/test_http_resolver_cache.py | 4 +--- tests/http_resolver/test_http_resolver_debug.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/http_resolver/test_http_resolver_cache.py b/tests/http_resolver/test_http_resolver_cache.py index 443aed07..96209b2b 100644 --- a/tests/http_resolver/test_http_resolver_cache.py +++ b/tests/http_resolver/test_http_resolver_cache.py @@ -82,7 +82,6 @@ def test_sign_with_cache(self): key = f.read() # ta_url is None, meaning "no timestamp authority". - # An empty string is treated as a URL and fails signing. signer_info = c2pa.C2paSignerInfo( alg=b"es256", sign_cert=certs, private_key=key, ta_url=None) @@ -153,9 +152,8 @@ def test_sign_with_cache(self): self.assertEqual(resolver.cache.misses, 1) self.assertEqual(resolver.cache.hits, 2) - def test_failing_resolver_is_a_clean_error(self): + def test_failing_resolver_is_reported_as_error(self): """A final failure surfaces as a typed error, not a crash. - Needs no network: the resolver answers without calling out. """ with c2pa.Context.builder().with_resolver( diff --git a/tests/http_resolver/test_http_resolver_debug.py b/tests/http_resolver/test_http_resolver_debug.py index cdf770fa..fb1d9b95 100644 --- a/tests/http_resolver/test_http_resolver_debug.py +++ b/tests/http_resolver/test_http_resolver_debug.py @@ -133,8 +133,8 @@ def test_sign_with_resolver(self): any(m == "GET" for m, _ in resolver.requests)) requests_before_reread = len(resolver.requests) - # Reading the signed file back uses its embedded - # manifest, so this makes no HTTP requests at all. + # Reading the signed file back uses its embedded manifest, + # so this makes no HTTP requests at all. print("Re-reading the signed output " "(should trigger no HTTP requests)") with open(output_path, "rb") as f: From 17adc388be62cf7f55120f84fe5f15404ad4a0d3 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:42:59 -0700 Subject: [PATCH 61/67] fix: Docs clean up once more 2 --- tests/http_resolver/http_resolver_example_impl.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/http_resolver/http_resolver_example_impl.py b/tests/http_resolver/http_resolver_example_impl.py index 9098a68d..70924460 100644 --- a/tests/http_resolver/http_resolver_example_impl.py +++ b/tests/http_resolver/http_resolver_example_impl.py @@ -123,7 +123,6 @@ def __init__(self, max_items=100, ttl_seconds=120.0): def get(self, key): with self._lock: entry = self._entries.get(key) - # time.monotonic, not time.time: immune to wall-clock jumps. if entry is not None and entry[0] > time.monotonic(): self._entries.move_to_end(key) self.hits += 1 @@ -192,8 +191,7 @@ def _fetch_with_retries(self, request): except urllib.error.HTTPError as e: retryable = e.code in (429, 503) if not retryable or attempt == self._max_retries: - # Final failure: pass the status back so the SDK can - # report it. + # Pass the status back so the SDK can report it. return e.code, e.read() delay = self._retry_delay(e, attempt) time.sleep(delay) @@ -232,7 +230,7 @@ def resolve(self, request): _reject_non_http(request.url) self.requests.append((request.method, request.url)) - # Timestamp requests POST a body; manifest fetches send none. + # Timestamp requests POST a body, manifest fetches send none. data = request.body or None req = urllib.request.Request( request.url, @@ -244,7 +242,7 @@ def resolve(self, request): with urllib.request.urlopen(req, timeout=self._timeout) as resp: return HttpResponse(resp.status, resp.read()) except urllib.error.HTTPError as e: - # A 4xx/5xx is still a response! Pass the status through and - # let the SDK turn it into its own typed error: a remote - # manifest fetch only accepts 200. + # A 4xx/5xx is still a response. + # Pass the status through and let the SDK turn it into its own typed error: + # a remote manifest fetch only accepts 200 as marker the data was retrieved. return HttpResponse(e.code, e.read()) From 89d879cd7dd6194e1e0911ea24b8ea9e4b869a1d Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:44:23 -0700 Subject: [PATCH 62/67] fix: Docs clean up once more 3 --- tests/http_resolver/test_http_resolver_cache.py | 9 +-------- tests/http_resolver/test_http_resolver_debug.py | 10 +--------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/tests/http_resolver/test_http_resolver_cache.py b/tests/http_resolver/test_http_resolver_cache.py index 96209b2b..9fee19f3 100644 --- a/tests/http_resolver/test_http_resolver_cache.py +++ b/tests/http_resolver/test_http_resolver_cache.py @@ -11,14 +11,7 @@ # specific language governing permissions and limitations under # each license. -"""Tests for CachingHttpResolver -(tests/http_resolver/http_resolver_example_impl.py), -exercised against the real network. - -Needs internet access to fetch the remote manifest for -tests/fixtures/cloud.jpg: these tests fail without it. On a Python -install with no CA bundle configured, run them with -SSL_CERT_FILE=$(python -m certifi). +"""Tests for CachingHttpResolver, an HTTP resolver that caches on read. """ import os diff --git a/tests/http_resolver/test_http_resolver_debug.py b/tests/http_resolver/test_http_resolver_debug.py index fb1d9b95..27da1e72 100644 --- a/tests/http_resolver/test_http_resolver_debug.py +++ b/tests/http_resolver/test_http_resolver_debug.py @@ -11,15 +11,7 @@ # specific language governing permissions and limitations under # each license. -"""Tests for DebugHttpResolver -(tests/http_resolver/http_resolver_example_impl.py), exercised -against the real network. - -Demonstrates intercepting every HTTP request the SDK makes through a -Context. Needs internet access to fetch the remote manifest for -tests/fixtures/cloud.jpg: these tests fail without it. On a Python -install with no CA bundle configured, run them with -SSL_CERT_FILE=$(python -m certifi). +"""Tests for DebugHttpResolver, an HTTP resolver that logs requests, """ import json From d140c172f2f7b25fadf8143bdd42e792e8751c02 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 20:06:45 -0700 Subject: [PATCH 63/67] fix: Docs pass --- tests/http_resolver/README.md | 154 +++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 39 deletions(-) diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index 717bfb40..49ccd8df 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -1,17 +1,17 @@ # Custom HTTP resolvers -This document explains how (custom) HTTP resolvers works end to end, what the SDK does and does not do with HTTP, and the platform differences (Linux, Windows, macOS) that affect a resolver in practice. +This document explains how (custom) HTTP resolvers work end to end, what the SDK does and does not do with HTTP, and the platform differences (Linux, Windows, macOS) that affect a resolver in practice. ## HTTP resolvers overview -A custom HTTP resolver lets you intercept every HTTP request the SDK makes through a `Context`, so you can add headers, cache responses, or serve responses from memory in tests. +A custom HTTP resolver intercepts every HTTP request the SDK makes through a `Context`, so an application can add headers, cache responses, or serve responses from memory in tests. c2pa ships no resolver types at all: there is no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `ContextBuilder.with_resolver()` (and the `Context(resolver=...)` constructor argument, also accepted by `Context.from_json`/`from_dict`) never does an `isinstance` check. It accepts either: - any callable taking one request argument; - any object with a `resolve(request)` method. -The request handed to you exposes `.url` (str), `.method` (str), `.headers` (dict), and `.body` (bytes). The value you return only needs `.status` (int) and `.body` (bytes) attributes. The minimal form needs no imports at all: +The request passed to the custom resolver exposes `.url` (str), `.method` (str), `.headers` (dict), and `.body` (bytes). The value it returns only needs `.status` (int) and `.body` (bytes) attributes. The minimal form needs no imports at all: ```py def my_resolver(request): @@ -24,11 +24,48 @@ reader = c2pa.Reader("image/jpeg", stream, context=context) The resolver is validated immediately: something with a wrong shape raises `TypeError` from `with_resolver()` itself (or from the `Context` constructor on the direct path), not later at `.build()`. If a resolver object has both a `resolve()` method and is itself callable, `resolve()` wins. -[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class (subclassing it is not required, it just gets you a documented, type-checkable contract instead of duck typing), plus three example resolvers: `DebugHttpResolver` (logs every request/response, delegates the transfer to `urllib`), `CachingHttpResolver` (TTL'd LRU cache plus retry/backoff for throttled requests), and `AlwaysFailResolver` (answers every request with a fixed status; needs no network). The module has no dependency on `c2pa` itself; only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. +[`http_resolver_example_impl.py`](./http_resolver_example_impl.py) is an example implementation of that shape. It defines `HttpRequest`, `HttpResponse`, and an optional `HttpResolver` abstract base class (subclassing it is not required, as it just provides a documented, type-checkable contract instead of duck typing), and three example resolvers: + +- `DebugHttpResolver`: logs every request/response, delegates the transfer to `urllib`. +- `CachingHttpResolver`: TTL'd LRU cache plus retry/backoff for throttled requests. +- `AlwaysFailResolver`: answers every request with a fixed status and needs no network. + +The module has no dependency on `c2pa` itself. Only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. + +## How the SDK uses the network (and why http or https) + +The SDK asks the resolver whenever verifying or signing needs a remote resource. + +Four kinds of request flow through, all triggered by reading or signing an asset: + +- Remote manifest fetch: a `GET` for a manifest stored outside the asset (the asset carries only a URL). Happens while reading, and while signing when an ingredient has a remote manifest. +- OCSP revocation: a request that checks whether the signing certificate has been revoked. The URL comes from the certificate's Authority Information Access extension, not from the custom resolver. +- RFC 3161 timestamps: a `POST` to a timestamp authority while signing, so the signature carries a trusted time. +- CAWG `did:web` resolution: a `GET` for the DID document backing an identity assertion, when a manifest uses CAWG identity. + +One HTTP(S) resolver, attached to one `Context`, is where all four pass through: + +```mermaid +flowchart LR + RB[Reader / Builder] --> C[Context] + C --> RES[Custom resolver] + RES --> M[Remote manifest GET] + RES --> O[OCSP revocation] + RES --> T[RFC 3161 timestamp POST] + RES --> D[CAWG did:web GET] +``` + +### Why http or why https + +The SDK does not pick the scheme to use, since it uses a URL it sees in the data and uses the protocol suggested by the URL (HTTP or HTTPS). It uses whatever the URL carries, and that URL comes from outside the SDK: the manifest's remote reference, the certificate's OCSP URL, the configured timestamp authority, or the DID. The scheme is a property of the endpoint, not a setting the application controls. + +Manifest fetches, timestamps, and `did:web` are normally `https`. OCSP is often plain `http`: an OCSP response is itself CMS-signed, so its integrity does not depend on TLS, and fetching revocation over `https` risks a circular dependency (validating a certificate would require validating the OCSP responder's own certificate first). An `http` OCSP URL sitting next to `https` everywhere else is normal, not a downgrade. + +Because the URL can be tweaked (it can come from whatever asset a caller supplies), a custom resolver should reject schemes it does not expect before making any request. The two network-facing examples, `DebugHttpResolver` and `CachingHttpResolver`, reject any URL whose scheme is not `http`/`https` for this reason (`AlwaysFailResolver` makes no request, so it does not). See the host-filtering note under [What the SDK leaves to the resolver](#what-the-sdk-leaves-to-the-resolver). ## Notes on runtime -The native library and the Python process must share a C runtime: buffers the HTTP resolver bridge allocates are freed on the native side with `libc::free`, so a native library built against a different C runtime (e.g. a static-musl `.so`) than the one your Python process loads will corrupt memory. +The native library and the Python process must share a C runtime: buffers the HTTP resolver bridge allocates are freed on the native side with `libc::free`, so a native library built against a different C runtime (e.g. a static-musl `.so`) than the one the Python process loads will corrupt memory. ## What traffic flows through a resolver (and when) @@ -36,87 +73,120 @@ A resolver attached to a `Context` handles **every** HTTP request the SDK makes - The resolver is **per-Context**. `Reader` and `Builder` instances created *without* that `Context` keep using the SDK's built-in resolver. Attaching a resolver to one `Context` changes nothing anywhere else. - `with_resolver()` can be called multiple times when creating a context to use: the last resolver set wins and will be used. -- **Settings still gate the resolver.** With `verify.remote_manifest_fetch` set to `false`, the resolver is never invoked for a remote-manifest read; the read fails instead. A resolver is not a way to re-enable disabled fetching. +- **Settings still gate the resolver.** With `verify.remote_manifest_fetch` set to `false`, the resolver is never invoked for a remote-manifest read. The read fails instead. A resolver is not a way to re-enable disabled fetching. ## How resolution works, end to end -Conceptually the pipeline is: native code decides it needs an HTTP resource, calls back into Python, your resolver performs the transfer however it likes, then the response is copied back into native memory. Concretely: +Conceptually the pipeline is: native code decides it needs an HTTP resource, calls back into Python, the custom resolver performs the transfer however it likes, then the response is copied back into native memory. + +The **trampoline** is the small ctypes callback that bridges the two sides. It is a function (built by `C2paHttpResolverBridge._make_trampoline`) that native code invokes through a raw C function pointer, and whose only job is to bounce that call into Python: decode the native request, call the custom resolver's `resolve()`, and encode the result back into native memory. It is a thin shim that exists only to redirect a call from one calling convention (here, the C ABI) into another (a Python callable). The call bounces off it into Python and the result bounces back across the C FFI boundary. It adapts between the two sides but does none of the HTTP work itself. -1. **Wiring.** `Context.__init__` normalizes your resolver into a plain callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver`; the native context builder takes ownership, so there is nothing for you to free. -2. **Invocation.** Whenever the native library needs an HTTP resource through that context, it synchronously invokes the trampoline with a pointer to a native request struct and a pointer to a zero-initialized native response struct. The call blocks the SDK operation (`Reader(...)`, `builder.sign(...)`, `add_ingredient(...)`) that triggered it, and **there is no SDK-side timeout**. A resolver that hangs, hangs that SDK call. Timeouts are entirely your responsibility (both example resolvers pass `timeout=` to `urllib.request.urlopen`). -3. **Decode.** The trampoline copies everything out of the native request struct (URL, method, headers, body) into a plain Python object (`C2paHttpRequestData`) holding `str`/`bytes`. Because it is a copy, the request object stays valid after your `resolve()` returns; storing it (as `DebugHttpResolver` stores `(method, url)` tuples) is safe. -4. **Resolve.** Your `resolve()` runs and returns a response-shaped object, or raises. -5. **Encode.** The trampoline validates the response (`.status` must be an `int`, `.body` must be `bytes`/`bytearray`), copies a non-empty body into a buffer allocated with the C runtime `malloc`, and writes status/body/length into the native response struct. Ownership of that buffer transfers to native code, which frees it on both the success and the error path. You never allocate or free anything yourself; you only ever return `bytes`. +Concretely, step by step: + +1. **Wiring.** `Context.__init__` normalizes the custom resolver into a plain callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver`. The native context builder takes ownership, so there is nothing for the caller to free. +2. **Invocation.** Whenever the native library needs an HTTP resource through that context, it synchronously invokes the trampoline with a pointer to a native request struct and a pointer to a zero-initialized native response struct. The call blocks the SDK operation (`Reader(...)`, `builder.sign(...)`, `add_ingredient(...)`) that triggered it, and **there is no SDK-side timeout**. A resolver that hangs, hangs that SDK call. Timeouts are entirely the custom resolver's responsibility (the network-facing examples pass `timeout=` to `urllib.request.urlopen`). +3. **Decode.** The trampoline copies everything out of the native request struct (URL, method, headers, body) into a plain Python object (`C2paHttpRequestData`) holding `str`/`bytes`. Because it is a copy, the request object stays valid after `resolve()` returns. Storing it (as `DebugHttpResolver` stores `(method, url)` tuples) is safe. +4. **Resolve.** The custom resolver's `resolve()` runs and returns a response-shaped object, or raises. +5. **Encode.** The trampoline validates the response (`.status` must be an `int`, `.body` must be `bytes`/`bytearray`), copies a non-empty body into a buffer allocated with the C runtime `malloc`, and writes status/body/length into the native response struct. Ownership of that buffer transfers to native code, which frees it on both the success and the error path. The custom resolver never allocates or frees anything. It only ever returns `bytes`. 6. **Consume.** The native side copies the body out, frees the buffer, and hands the response to whatever validation or signing logic asked for it. +```mermaid +sequenceDiagram + participant N as SDK native library + participant T as ctypes trampoline + participant R as Custom resolve() + N->>T: request*, response* (zero-init) + T->>T: copy url/method/headers/body to Python + T->>R: resolve(request) + R-->>T: response (.status int, .body bytes) + T->>T: validate, then malloc + copy body + T-->>N: write status/body/len, return 0 + N->>N: copy body out, then free() the buffer +``` + ## Request semantics - `url` is the absolute request URL. - `method` is the HTTP verb: `GET` for manifest fetches, `POST` for timestamp requests. -- `headers` is a dict. Header names arrive **lowercased** by the native layer. Repeated headers are delivered as separate lines internally; in the dict, the **last occurrence wins**. -- `body` is `b""` when there is no body (manifest fetches); timestamp requests `POST` a body. `request.body or None` is the idiomatic way to hand it to `urllib.request.Request`, as both examples do. +- `headers` is a dict. Header names arrive **lowercased** by the native layer. Repeated headers are delivered as separate lines internally. In the dict, the **last occurrence wins**. +- `body` is `b""` when there is no body (manifest fetches). Timestamp requests `POST` a body. `request.body or None` is the idiomatic way to hand it to `urllib.request.Request`, as the network-facing examples do. ## Response and error semantics -- **Status passes through.** Return the real status; do not translate. For a remote manifest fetch, only `200` is accepted; anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. -- **Raising marks a hard failure.** A transport-level problem (DNS failure, connection refused, timeout) is not a response; raise, and the SDK reports the request as failed. The examples deliberately do *not* catch `urllib.error.URLError` for exactly this reason. -- **Your exception does not propagate as itself.** Exceptions cannot unwind across the ctypes/native boundary. The trampoline catches everything you raise, including `BaseException` and `KeyboardInterrupt`, records its message in the native error slot, and the failure re-emerges as a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch. `except MyCustomError:` around `c2pa.Reader(...)` will never fire; catch `c2pa.C2paError` and read the message. -- **Shape errors are caught early.** Returning a `str` body, a `None` status, or a `bool` status is rejected inside the trampoline with a clear `TypeError` message, which then surfaces the same way (as a `C2paError`). +- **Status passes through.** Return the real status, and do not translate. For a remote manifest fetch, only `200` is accepted. Anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. +- **Raising marks a hard failure.** A transport-level problem (DNS failure, connection refused, timeout) is not a response. Raise, and the SDK reports the request as failed. The examples deliberately do *not* catch `urllib.error.URLError` for exactly this reason. +- **A raised exception does not propagate as itself.** Exceptions cannot unwind across the ctypes/native boundary. The trampoline catches everything the custom resolver raises, including `BaseException` and `KeyboardInterrupt`, records its message in the native error slot, and the failure re-emerges as a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch. An `except MyCustomError:` around `c2pa.Reader(...)` will never fire. Callers should catch `c2pa.C2paError` and read the message. +- **Shape errors are caught early.** Returning a `str` body, a `None` or `bool` status, or a status outside the 100-599 range is rejected inside the trampoline with a clear `TypeError` message, which then surfaces the same way (as a `C2paError`). - An **empty body** is fine: return `b""` (as `AlwaysFailResolver` does), and the trampoline correctly leaves the native body pointer/length pair empty together. ## Lifetimes -- **The resolver outlives `Context.close()`.** Native `Reader`/`Builder` instances hold their own reference to the underlying native context, so your resolver can still be invoked after the Python `Context` is closed, for as long as any `Reader` or `Builder` created from it is alive. Do not tear down resolver resources (close a session, release a pool) on `Context.close()`; tie them to the resolvers' own lifetime instead. The SDK internally pins the callback thunk to keep this safe, so there is nothing you need to hold onto yourself. -- **No reentrancy.** Do not call c2pa APIs from inside `resolve()`. Re-entering the FFI while a call is in flight is undefined. -- **Concurrent close() is unsafe.** Do not close a `Context`/`Reader`/`Builder` from one thread while another thread still has a call in flight on that same object, including a resolver call it triggered. This is a general property of these objects, not something specific to resolvers, but resolver users are the most likely to be multi-threaded. +- **The resolver outlives `Context.close()`.** Native `Reader`/`Builder` instances hold their own reference to the underlying native context, so the custom resolver can still be invoked after the Python `Context` is closed, for as long as any `Reader` or `Builder` created from it is alive. A custom resolver must not tear down its resources (close a session, release a pool) on `Context.close()`. Those should be tied to the resolver's own lifetime instead. The SDK internally pins the callback thunk to keep this safe, so there is nothing the caller needs to hold onto. +- **No reentrancy.** A custom resolver must not call c2pa APIs from inside `resolve()`. Re-entering the FFI while a call is in flight is undefined. +- **Do not close while a call is in flight.** A `Context`/`Reader`/`Builder` must not be closed while a call is still running on that same object, including a resolver call it triggered. This is a general property of these objects, not something specific to resolvers. -## What the SDK does and does not do with HTTP +## What the SDK leaves to the resolver The SDK delegates the *transfer* entirely; the resolver *is* the HTTP client. That means: -- **No redirects.** The SDK does not follow redirects; a `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as both examples do) gives you redirect handling for free. A hand-rolled resolver must implement it. -- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; if you need an allowlist, enforce it yourself in `resolve()` (raise or return an error status for disallowed hosts). The URL is attacker-influenced: it comes from a remote-manifest reference embedded in whatever asset someone hands the application, so a resolver that hands it straight to a general-purpose HTTP client without a scheme check is a `file://` local-file-read and internal-network SSRF gadget. Both example resolvers reject any URL whose scheme is not `http`/`https` before doing anything else, for this reason. -- **TLS is yours.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack your resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives; see the platform section below. -- **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. If you serve large manifests, be aware the failure mode downstream is a validation error, not a size error. -- **No caching, retries, or backoff.** Each needed resource is requested; policy is yours. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. +- **No redirects.** The SDK does not follow redirects. A `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as the network-facing examples do) gives redirect handling. Make sure to implement (or block) redirects as needed by the custom resolver. +- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; one that needs an allowlist must enforce it in `resolve()` (raise or return an error status for disallowed hosts). The URL comes from a remote-manifest reference embedded in the asset. The network-facing examples reject any URL whose scheme is not `http`/`https` before doing anything else. Validating the host (and, depending on the deployment, the resolved address and port) is worth considering too. +- **TLS belongs to the resolver.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack the custom resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives. See the platform section below. +- **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. A resolver that serves large manifests should note the failure mode downstream is a validation error, not a size error. +- **No caching, retries, or backoff.** Each needed resource is requested; policy belongs to the custom resolver. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. ## Platform differences: Linux, Windows, macOS -The trampoline itself behaves identically everywhere, but four areas differ per platform in ways that bite resolver implementers. +The trampoline itself behaves identically everywhere, but three areas differ per platform in ways that bite resolver implementers. -### 1. The C runtime and the response buffer (why you never allocate) +### 1. The C runtime and the response buffer (why a resolver never allocates) The response body buffer must be allocated by the **same C runtime whose `free()` the native library calls** (on the Rust side that is `libc::free`). On Linux and macOS there is effectively one C runtime per process (glibc/musl, libSystem), so "the process's own libc" is always the right allocator. **Windows is different:** a process can host several C runtimes side by side, each with its own heap. Rust's MSVC targets link the Universal CRT, so `free` there is `ucrtbase`'s, while the legacy `msvcrt.dll` is a *different* heap. Allocating from one and freeing into the other is heap corruption, not a leak, and it corrupts silently until it crashes somewhere unrelated. -The bridge handles this for you: it loads `ucrtbase` first and falls back to `msvcrt`, and does the `malloc`+copy from the `bytes` you return. The rule for implementers: **return `bytes`, never a pointer, never something you allocated with ctypes yourself.** The reason this rule exists is Windows. +This is the reason the bindings carry `ManagedResource._get_native_malloc()` rather than allocating the response buffer from Python's own memory or a plain `ctypes` call. The helper resolves, once and lazily, the exact `malloc` whose heap matches the native library's `free`: + +- On Windows it loads `ucrtbase` first and falls back to `msvcrt`, matching the CRT that Rust's `libc::free` uses. +- On Linux and macOS it opens the process's own C library with `ctypes.CDLL(None)`, so its `malloc` and the native `free` come from the same libc. + +The helper also sets `malloc.restype = ctypes.c_void_p`. Without that, `ctypes` assumes a `c_int` return and truncates a 64-bit pointer to 32 bits. The truncated address handed to native `free` is a second, quieter way to corrupt the heap. The looked-up function is cached on the class, so the resolution happens at most once per process. + +The trampoline calls that `malloc`, copies the returned `bytes` into the buffer, and writes the pointer into the native response struct, and native code frees it afterwards. That is the whole reason a custom resolver returns `bytes` and nothing else: **it returns `bytes`, never a pointer, and never memory it allocated with `ctypes` itself.** The reason this rule exists is Windows. + +The body buffer's ownership follows one path: + +```mermaid +flowchart TD + A[resolve returns bytes] --> B{body empty?} + B -- yes --> C[native body = NULL, len = 0
nothing allocated, nothing to free] + B -- no --> D[trampoline malloc + memmove] + D --> E[write pointer and len into response struct] + E --> F[ownership transfers to native] + F --> G[native copies body out, then calls free] +``` ### 2. TLS trust stores The example resolvers delegate to `urllib`, so they inherit Python's `ssl` defaults, which differ by platform: - **macOS:** python.org builds do **not** use the system Keychain. If `Install Certificates.command` was never run after installing Python, every HTTPS fetch fails with `CERTIFICATE_VERIFY_FAILED`. Workaround without reinstalling: prefix the run with `SSL_CERT_FILE=$(python -m certifi)`. -- **Linux:** OpenSSL uses the distribution's CA bundle. On a normal desktop this just works; in slim container images the `ca-certificates` package is often missing, producing the same `CERTIFICATE_VERIFY_FAILED`. Install the package or set `SSL_CERT_FILE`. +- **Linux:** OpenSSL uses the distribution's CA bundle. On a normal desktop this just works. In slim container images the `ca-certificates` package is often missing, producing the same `CERTIFICATE_VERIFY_FAILED`. Install the package or set `SSL_CERT_FILE`. - **Windows:** Python's `ssl` loads roots from the Windows certificate store, so system-managed (including enterprise-injected) roots are honored automatically. Corporate TLS-interception proxies therefore tend to *work* on Windows and *fail* on macOS/Linux with the same code. If a fetch verifies on one machine and not another, compare trust stores before suspecting the resolver. A resolver using a different HTTP stack has that stack's trust behavior instead. TLS trust is resolver-side, per-platform, and invisible to the SDK. ### 3. Proxies -`urllib` discovers proxies differently per platform: environment variables (`http_proxy`, `https_proxy`, `no_proxy`) everywhere, **plus** the Windows registry (Internet Settings) on Windows and the System Configuration framework (Network preferences) on macOS. On Linux, environment variables are the only source. So a resolver built on `urllib` silently follows OS-level proxy settings on Windows/macOS but ignores them on Linux, another way the same resolver code behaves differently per machine. If you need deterministic behavior, configure the proxy (or the absence of one) explicitly in your resolver rather than relying on discovery. - -### 4. Process model: fork vs. spawn - -Default `multiprocessing` start methods differ: historically `fork` on Linux (`forkserver` since Python 3.14), `spawn` on macOS and Windows. Under `spawn`, a child process imports fresh and never inherits a `Context`, so there is nothing to think about. Under `fork`, the child inherits the parent's memory image, including a `Context` with a resolver attached. The SDK's native handles are PID-stamped so a forked child neither uses nor frees the parent's native resources (see [`../../docs/native-resources-management.md`](../../docs/native-resources-management.md)), but your *resolver's own* state is ordinary Python and gets copied: locks are cloned in whatever state they were in, background threads do **not** survive the fork, and open sockets/sessions are shared with the parent. A resolver holding only per-call state (like both examples) forks safely; one holding live connections should be recreated in the child. This concern is Linux-only in practice. +`urllib` discovers proxies differently per platform: environment variables (`http_proxy`, `https_proxy`, `no_proxy`) everywhere, **plus** the Windows registry (Internet Settings) on Windows and the System Configuration framework (Network preferences) on macOS. On Linux, environment variables are the only source. So a resolver built on `urllib` silently follows OS-level proxy settings on Windows/macOS but ignores them on Linux, another way the same resolver code behaves differently per machine. For deterministic behavior, a custom resolver should configure the proxy (or its absence) explicitly rather than relying on discovery. ## Examples and tests -- [`http_resolver_example_impl.py`](./http_resolver_example_impl.py): the reference shapes (`HttpRequest`, `HttpResponse`, optional `HttpResolver` ABC) and the three resolvers described above. Copy it into your project and adapt it; it does not import `c2pa`. +- [`http_resolver_example_impl.py`](./http_resolver_example_impl.py): the reference shapes (`HttpRequest`, `HttpResponse`, optional `HttpResolver` ABC) and the three resolvers described above. Copy it into a project and adapt it. It does not import `c2pa`. - [`test_http_resolver_debug.py`](./test_http_resolver_debug.py): exercises `DebugHttpResolver`, verifying that a remote-manifest read logs a `GET`, that signing with a remote-manifest ingredient fetches it through the resolver, and that re-reading the signed (embedded-manifest) output performs no HTTP at all. - [`test_http_resolver_cache.py`](./test_http_resolver_cache.py): exercises `CachingHttpResolver` (reading twice / ingesting the same ingredient repeatedly hits the cache exactly as the hit/miss counters predict) and `AlwaysFailResolver` (a non-200 answer surfaces as a clean typed `C2paError`, with no network needed). Those examples need network access to fetch the remote manifest for `tests/fixtures/cloud.jpg`, and fail without it. -If your Python has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above); on macOS python.org builds this is the common case, so the commands below include the fix. +If the Python runtime has no CA bundle configured, every fetch fails with `CERTIFICATE_VERIFY_FAILED` (see the TLS section above). On macOS python.org builds this is the common case, so the commands below include the fix. Run them in a CLI with the commands: @@ -128,7 +198,7 @@ python ./tests/http_resolver/test_http_resolver_cache.py python -m unittest discover -s tests/http_resolver -v ``` -On some operating systems, if you hit CERTIFICATE_VERIFY_FAILED, this prefix on the commands fixes it: +On some operating systems, if a CERTIFICATE_VERIFY_FAILED error appears, this prefix on the commands fixes it: ```bash SSL_CERT_FILE=$(python -m certifi) python ./tests/http_resolver/test_http_resolver_debug.py @@ -137,3 +207,9 @@ SSL_CERT_FILE=$(python -m certifi) python ./tests/http_resolver/test_http_resolv # Or the whole directory via unittest discovery: SSL_CERT_FILE=$(python -m certifi) python -m unittest discover -s tests/http_resolver -v ``` + +## See also + +- [`http_resolver_example_impl.py`](./http_resolver_example_impl.py) — copyable reference resolvers (`DebugHttpResolver`, `CachingHttpResolver`, `AlwaysFailResolver`). +- [`../../docs/context-settings.md`](../../docs/context-settings.md) — settings that gate networking, including `core.allowed_network_hosts` and `verify.remote_manifest_fetch`. +- [`../../docs/native-resources-management.md`](../../docs/native-resources-management.md) — how the SDK manages native handles and their lifetimes, including behavior across process boundaries. From 24b9343e1fe2d4733896e3398b5debc9e3f87a62 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 20:18:33 -0700 Subject: [PATCH 64/67] fix: Docs pass 2 --- tests/http_resolver/README.md | 40 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index 49ccd8df..0e48fc27 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -1,8 +1,8 @@ -# Custom HTTP resolvers +# Custom HTTP resolver examples This document explains how (custom) HTTP resolvers work end to end, what the SDK does and does not do with HTTP, and the platform differences (Linux, Windows, macOS) that affect a resolver in practice. -## HTTP resolvers overview +## Overview A custom HTTP resolver intercepts every HTTP request the SDK makes through a `Context`, so an application can add headers, cache responses, or serve responses from memory in tests. @@ -32,7 +32,7 @@ The resolver is validated immediately: something with a wrong shape raises `Type The module has no dependency on `c2pa` itself. Only the tests import `c2pa`, to exercise the resolvers against real `Context`/`Reader`/`Builder` instances. -## How the SDK uses the network (and why http or https) +## How the SDK uses the network The SDK asks the resolver whenever verifying or signing needs a remote resource. @@ -75,20 +75,36 @@ A resolver attached to a `Context` handles **every** HTTP request the SDK makes - `with_resolver()` can be called multiple times when creating a context to use: the last resolver set wins and will be used. - **Settings still gate the resolver.** With `verify.remote_manifest_fetch` set to `false`, the resolver is never invoked for a remote-manifest read. The read fails instead. A resolver is not a way to re-enable disabled fetching. -## How resolution works, end to end +## How resolution works end to end -Conceptually the pipeline is: native code decides it needs an HTTP resource, calls back into Python, the custom resolver performs the transfer however it likes, then the response is copied back into native memory. +The pipeline is: native code decides it needs an HTTP resource, calls back into Python, the custom resolver performs the transfer however it likes, then the response is copied back into native memory. -The **trampoline** is the small ctypes callback that bridges the two sides. It is a function (built by `C2paHttpResolverBridge._make_trampoline`) that native code invokes through a raw C function pointer, and whose only job is to bounce that call into Python: decode the native request, call the custom resolver's `resolve()`, and encode the result back into native memory. It is a thin shim that exists only to redirect a call from one calling convention (here, the C ABI) into another (a Python callable). The call bounces off it into Python and the result bounces back across the C FFI boundary. It adapts between the two sides but does none of the HTTP work itself. +```mermaid +flowchart LR + subgraph N[Native side, C / Rust] + direction TB + SDK[SDK needs an HTTP resource] + DONE[Response in native memory] + end + subgraph P[Python side] + RES["Custom resolver: resolve()"] + end + SDK -->|1. request| TR(["trampoline
(ctypes callback)"]) + TR -->|2. decoded request| RES + RES -->|3. status + body bytes| TR + TR -->|4. encoded response| DONE +``` + +The "trampoline" is the small ctypes callback that bridges the two sides. It is a function (built by `C2paHttpResolverBridge._make_trampoline`) that native code invokes through a raw C function pointer, and whose only job is to bounce that call into Python: decode the native request, call the custom resolver's `resolve()`, and encode the result back into native memory. It is a thin shim that exists only to redirect a call from one calling convention (here, the C ABI) into another (a Python callable). The call bounces off it into Python and the result bounces back across the C FFI boundary. It adapts between the two sides but does none of the HTTP work itself. -Concretely, step by step: +Step by step: -1. **Wiring.** `Context.__init__` normalizes the custom resolver into a plain callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver`. The native context builder takes ownership, so there is nothing for the caller to free. +1. **Wiring.** `Context.__init__` normalizes the custom resolver into a callable (the `resolve` bound method, or the callable itself) and wraps it in a ctypes trampoline (`C2paHttpResolverBridge._make_trampoline`). A short-lived native resolver handle is created via `c2pa_http_resolver_create` and consumed by `c2pa_context_builder_set_http_resolver`. The native context builder takes ownership, so there is nothing for the caller to free. 2. **Invocation.** Whenever the native library needs an HTTP resource through that context, it synchronously invokes the trampoline with a pointer to a native request struct and a pointer to a zero-initialized native response struct. The call blocks the SDK operation (`Reader(...)`, `builder.sign(...)`, `add_ingredient(...)`) that triggered it, and **there is no SDK-side timeout**. A resolver that hangs, hangs that SDK call. Timeouts are entirely the custom resolver's responsibility (the network-facing examples pass `timeout=` to `urllib.request.urlopen`). 3. **Decode.** The trampoline copies everything out of the native request struct (URL, method, headers, body) into a plain Python object (`C2paHttpRequestData`) holding `str`/`bytes`. Because it is a copy, the request object stays valid after `resolve()` returns. Storing it (as `DebugHttpResolver` stores `(method, url)` tuples) is safe. 4. **Resolve.** The custom resolver's `resolve()` runs and returns a response-shaped object, or raises. 5. **Encode.** The trampoline validates the response (`.status` must be an `int`, `.body` must be `bytes`/`bytearray`), copies a non-empty body into a buffer allocated with the C runtime `malloc`, and writes status/body/length into the native response struct. Ownership of that buffer transfers to native code, which frees it on both the success and the error path. The custom resolver never allocates or frees anything. It only ever returns `bytes`. -6. **Consume.** The native side copies the body out, frees the buffer, and hands the response to whatever validation or signing logic asked for it. +6. **Consume.** The native side copies the body out, frees the buffer, and hands the response to whatever logic asked for it. ```mermaid sequenceDiagram @@ -137,15 +153,15 @@ The SDK delegates the *transfer* entirely; the resolver *is* the HTTP client. Th ## Platform differences: Linux, Windows, macOS -The trampoline itself behaves identically everywhere, but three areas differ per platform in ways that bite resolver implementers. +Three things can affect the resolver depending on the platform. ### 1. The C runtime and the response buffer (why a resolver never allocates) -The response body buffer must be allocated by the **same C runtime whose `free()` the native library calls** (on the Rust side that is `libc::free`). On Linux and macOS there is effectively one C runtime per process (glibc/musl, libSystem), so "the process's own libc" is always the right allocator. **Windows is different:** a process can host several C runtimes side by side, each with its own heap. Rust's MSVC targets link the Universal CRT, so `free` there is `ucrtbase`'s, while the legacy `msvcrt.dll` is a *different* heap. Allocating from one and freeing into the other is heap corruption, not a leak, and it corrupts silently until it crashes somewhere unrelated. +The response body buffer must be allocated by the **same C runtime whose `free()` the native library calls** (on the native side that is `libc::free`). On Linux and macOS there is effectively one C runtime per process, so "the process's own libc" is always the right allocator. **Windows is different:** a process can host several C runtimes side by side, each with its own heap. The native library links the Universal CRT, so its `free` is `ucrtbase`'s, while the legacy `msvcrt.dll` is a *different* heap. Allocating from one and freeing into the other is heap corruption, and it corrupts silently until it crashes somewhere unrelated. This is the reason the bindings carry `ManagedResource._get_native_malloc()` rather than allocating the response buffer from Python's own memory or a plain `ctypes` call. The helper resolves, once and lazily, the exact `malloc` whose heap matches the native library's `free`: -- On Windows it loads `ucrtbase` first and falls back to `msvcrt`, matching the CRT that Rust's `libc::free` uses. +- On Windows it loads `ucrtbase` first and falls back to `msvcrt`, matching the CRT the native library's `free` uses. - On Linux and macOS it opens the process's own C library with `ctypes.CDLL(None)`, so its `malloc` and the native `free` come from the same libc. The helper also sets `malloc.restype = ctypes.c_void_p`. Without that, `ctypes` assumes a `c_int` return and truncates a 64-bit pointer to 32 bits. The truncated address handed to native `free` is a second, quieter way to corrupt the heap. The looked-up function is cached on the class, so the resolution happens at most once per process. From 52c41d82007496f6d94af4a5a7b57fe4fbf93d4e Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 28 Jul 2026 20:30:16 -0700 Subject: [PATCH 65/67] fix: Docs pass 3 --- tests/http_resolver/README.md | 38 ++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index 0e48fc27..7b936a13 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -8,8 +8,8 @@ A custom HTTP resolver intercepts every HTTP request the SDK makes through a `Co c2pa ships no resolver types at all: there is no `HttpRequest`, `HttpResponse`, or `HttpResolver` to import from `c2pa`. `ContextBuilder.with_resolver()` (and the `Context(resolver=...)` constructor argument, also accepted by `Context.from_json`/`from_dict`) never does an `isinstance` check. It accepts either: -- any callable taking one request argument; -- any object with a `resolve(request)` method. +- any callable taking one request argument +- any object with a `resolve(request)` method The request passed to the custom resolver exposes `.url` (str), `.method` (str), `.headers` (dict), and `.body` (bytes). The value it returns only needs `.status` (int) and `.body` (bytes) attributes. The minimal form needs no imports at all: @@ -57,15 +57,15 @@ flowchart LR ### Why http or why https -The SDK does not pick the scheme to use, since it uses a URL it sees in the data and uses the protocol suggested by the URL (HTTP or HTTPS). It uses whatever the URL carries, and that URL comes from outside the SDK: the manifest's remote reference, the certificate's OCSP URL, the configured timestamp authority, or the DID. The scheme is a property of the endpoint, not a setting the application controls. +The SDK does not pick the scheme. It uses whatever the URL carries (HTTP or HTTPS). That URL comes from outside the SDK: the manifest's remote reference, the certificate's OCSP URL, the configured timestamp authority, or the DID. The scheme is a property of the endpoint, not a setting the application controls. -Manifest fetches, timestamps, and `did:web` are normally `https`. OCSP is often plain `http`: an OCSP response is itself CMS-signed, so its integrity does not depend on TLS, and fetching revocation over `https` risks a circular dependency (validating a certificate would require validating the OCSP responder's own certificate first). An `http` OCSP URL sitting next to `https` everywhere else is normal, not a downgrade. +Manifest fetches, timestamps, and `did:web` are normally `https`. OCSP is often plain `http`. An OCSP response is itself CMS-signed, so its integrity does not depend on TLS. Fetching revocation over `https` also risks a circular dependency: validating a certificate would require validating the OCSP responder's own certificate first. An `http` OCSP URL sitting next to `https` everywhere else is normal, not a downgrade. Because the URL can be tweaked (it can come from whatever asset a caller supplies), a custom resolver should reject schemes it does not expect before making any request. The two network-facing examples, `DebugHttpResolver` and `CachingHttpResolver`, reject any URL whose scheme is not `http`/`https` for this reason (`AlwaysFailResolver` makes no request, so it does not). See the host-filtering note under [What the SDK leaves to the resolver](#what-the-sdk-leaves-to-the-resolver). ## Notes on runtime -The native library and the Python process must share a C runtime: buffers the HTTP resolver bridge allocates are freed on the native side with `libc::free`, so a native library built against a different C runtime (e.g. a static-musl `.so`) than the one the Python process loads will corrupt memory. +The native library and the Python process must share a C runtime. The bridge allocates buffers that the native side frees with `libc::free`. A native library built against a different C runtime than the one the Python process loads (for example a static-musl `.so`) will corrupt memory. ## What traffic flows through a resolver (and when) @@ -81,7 +81,7 @@ The pipeline is: native code decides it needs an HTTP resource, calls back into ```mermaid flowchart LR - subgraph N[Native side, C / Rust] + subgraph N[Native side] direction TB SDK[SDK needs an HTTP resource] DONE[Response in native memory] @@ -143,13 +143,13 @@ sequenceDiagram ## What the SDK leaves to the resolver -The SDK delegates the *transfer* entirely; the resolver *is* the HTTP client. That means: +The SDK delegates the *transfer* entirely. The resolver *is* the HTTP client. That means: - **No redirects.** The SDK does not follow redirects. A `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as the network-facing examples do) gives redirect handling. Make sure to implement (or block) redirects as needed by the custom resolver. -- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless; one that needs an allowlist must enforce it in `resolve()` (raise or return an error status for disallowed hosts). The URL comes from a remote-manifest reference embedded in the asset. The network-facing examples reject any URL whose scheme is not `http`/`https` before doing anything else. Validating the host (and, depending on the deployment, the resolved address and port) is worth considering too. +- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless. One that needs an allowlist must enforce it in `resolve()` (raise or return an error status for disallowed hosts). The URL comes from a remote-manifest reference embedded in the asset. The network-facing examples reject any URL whose scheme is not `http`/`https` before doing anything else. Validating the host (and, depending on the deployment, the resolved address and port) is worth considering too. - **TLS belongs to the resolver.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack the custom resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives. See the platform section below. - **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. A resolver that serves large manifests should note the failure mode downstream is a validation error, not a size error. -- **No caching, retries, or backoff.** Each needed resource is requested; policy belongs to the custom resolver. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. +- **No caching, retries, or backoff.** Each needed resource is requested. Policy belongs to the custom resolver. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. ## Platform differences: Linux, Windows, macOS @@ -168,16 +168,22 @@ The helper also sets `malloc.restype = ctypes.c_void_p`. Without that, `ctypes` The trampoline calls that `malloc`, copies the returned `bytes` into the buffer, and writes the pointer into the native response struct, and native code frees it afterwards. That is the whole reason a custom resolver returns `bytes` and nothing else: **it returns `bytes`, never a pointer, and never memory it allocated with `ctypes` itself.** The reason this rule exists is Windows. -The body buffer's ownership follows one path: +The response body crosses the Python/native boundary exactly once. Python fills the buffer and hands it over. Native code then frees it and passes the bytes to whatever needed the resource: ```mermaid flowchart TD - A[resolve returns bytes] --> B{body empty?} - B -- yes --> C[native body = NULL, len = 0
nothing allocated, nothing to free] - B -- no --> D[trampoline malloc + memmove] - D --> E[write pointer and len into response struct] - E --> F[ownership transfers to native] - F --> G[native copies body out, then calls free] + subgraph PY["Python side"] + A["custom resolver: resolve() returns bytes"] -->|handoff to the trampoline| B{body empty?} + B -- no --> D["trampoline: malloc + memmove into a C buffer"] + B -- yes --> C["trampoline: leave body = NULL, len = 0"] + D --> E["trampoline: write pointer + len into the response struct"] + C --> E + end + E ==>|ownership transfers to native| F + subgraph NA["Native side (SDK)"] + F["copy the body out of the buffer"] --> G["free the buffer"] + G --> H["hand the bytes to the consumer:
manifest validation, OCSP, timestamp, or did:web"] + end ``` ### 2. TLS trust stores From b0a3c6ef0d30b323071b2c617a4b06cee411bb66 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Wed, 29 Jul 2026 08:49:43 -0700 Subject: [PATCH 66/67] fix: Clean up docs and examples --- src/c2pa/c2pa.py | 3 + tests/http_resolver/README.md | 52 ++++++------- .../http_resolver_example_impl.py | 75 ++++++++++++++----- .../http_resolver/test_http_resolver_cache.py | 69 ++++++++++++++++- .../http_resolver/test_http_resolver_debug.py | 3 +- 5 files changed, 157 insertions(+), 45 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 19edc3de..207311ff 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1967,6 +1967,9 @@ def with_resolver( never invoked and the read fails instead. - Do not call any other c2pa APIs from inside the resolver: reentering the FFI while a call is in flight is undefined behavior. + - The resolver is retained for the Context's lifetime; avoid + having it capture the Context, Reader, or Builder (that forms + a reference cycle that delays native cleanup). """ self._resolve_fn = C2paHttpResolverBridge._coerce_resolver(resolver) self._resolver = resolver diff --git a/tests/http_resolver/README.md b/tests/http_resolver/README.md index 7b936a13..2b1bb85d 100644 --- a/tests/http_resolver/README.md +++ b/tests/http_resolver/README.md @@ -36,30 +36,20 @@ The module has no dependency on `c2pa` itself. Only the tests import `c2pa`, to The SDK asks the resolver whenever verifying or signing needs a remote resource. -Four kinds of request flow through, all triggered by reading or signing an asset: - -- Remote manifest fetch: a `GET` for a manifest stored outside the asset (the asset carries only a URL). Happens while reading, and while signing when an ingredient has a remote manifest. -- OCSP revocation: a request that checks whether the signing certificate has been revoked. The URL comes from the certificate's Authority Information Access extension, not from the custom resolver. -- RFC 3161 timestamps: a `POST` to a timestamp authority while signing, so the signature carries a trusted time. -- CAWG `did:web` resolution: a `GET` for the DID document backing an identity assertion, when a manifest uses CAWG identity. - -One HTTP(S) resolver, attached to one `Context`, is where all four pass through: +What that means in practice changes with the SDK version and configuration — a remote manifest fetch, a certificate revocation check, a trusted-timestamp request, and an identity-document lookup are all current examples, not a closed list. All of it is triggered by reading or signing an asset, and all of it funnels through the one resolver attached to the `Context`: ```mermaid flowchart LR RB[Reader / Builder] --> C[Context] C --> RES[Custom resolver] - RES --> M[Remote manifest GET] - RES --> O[OCSP revocation] - RES --> T[RFC 3161 timestamp POST] - RES --> D[CAWG did:web GET] + RES --> NET[Whatever HTTP resource the SDK currently needs] ``` ### Why http or why https -The SDK does not pick the scheme. It uses whatever the URL carries (HTTP or HTTPS). That URL comes from outside the SDK: the manifest's remote reference, the certificate's OCSP URL, the configured timestamp authority, or the DID. The scheme is a property of the endpoint, not a setting the application controls. +The SDK does not pick the scheme. It uses whatever the URL carries (HTTP or HTTPS). That URL comes from outside the SDK: a remote reference, a certificate extension, a configured authority, or similar. The scheme is a property of the endpoint, not a setting the application controls. -Manifest fetches, timestamps, and `did:web` are normally `https`. OCSP is often plain `http`. An OCSP response is itself CMS-signed, so its integrity does not depend on TLS. Fetching revocation over `https` also risks a circular dependency: validating a certificate would require validating the OCSP responder's own certificate first. An `http` OCSP URL sitting next to `https` everywhere else is normal, not a downgrade. +Most of what flows through is normally `https`. A certificate revocation check is a common example of one that can be plain `http` instead: its response is itself CMS-signed, so its integrity does not depend on TLS, and fetching it over `https` risks a circular dependency (validating a certificate would require validating the responder's own certificate first). An `http` URL sitting next to `https` everywhere else is normal, not a downgrade. Because the URL can be tweaked (it can come from whatever asset a caller supplies), a custom resolver should reject schemes it does not expect before making any request. The two network-facing examples, `DebugHttpResolver` and `CachingHttpResolver`, reject any URL whose scheme is not `http`/`https` for this reason (`AlwaysFailResolver` makes no request, so it does not). See the host-filtering note under [What the SDK leaves to the resolver](#what-the-sdk-leaves-to-the-resolver). @@ -123,13 +113,13 @@ sequenceDiagram ## Request semantics - `url` is the absolute request URL. -- `method` is the HTTP verb: `GET` for manifest fetches, `POST` for timestamp requests. +- `method` is the HTTP verb (`GET`, `POST`, ...); which verb shows up depends on what's being fetched. - `headers` is a dict. Header names arrive **lowercased** by the native layer. Repeated headers are delivered as separate lines internally. In the dict, the **last occurrence wins**. -- `body` is `b""` when there is no body (manifest fetches). Timestamp requests `POST` a body. `request.body or None` is the idiomatic way to hand it to `urllib.request.Request`, as the network-facing examples do. +- `body` is `b""` when there is no body. A request with a payload (typically a `POST`) carries one. `request.body or None` is the idiomatic way to hand it to `urllib.request.Request`, as the network-facing examples do. ## Response and error semantics -- **Status passes through.** Return the real status, and do not translate. For a remote manifest fetch, only `200` is accepted. Anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. +- **Status passes through.** Return the real status, and do not translate. Only `200` is treated as success; anything else surfaces to the SDK caller as a typed `C2paError`. `DebugHttpResolver` shows the right pattern for `urllib`: an `HTTPError` is still a response, so it returns `HttpResponse(e.code, e.read())` and lets the SDK produce its own error. - **Raising marks a hard failure.** A transport-level problem (DNS failure, connection refused, timeout) is not a response. Raise, and the SDK reports the request as failed. The examples deliberately do *not* catch `urllib.error.URLError` for exactly this reason. - **A raised exception does not propagate as itself.** Exceptions cannot unwind across the ctypes/native boundary. The trampoline catches everything the custom resolver raises, including `BaseException` and `KeyboardInterrupt`, records its message in the native error slot, and the failure re-emerges as a typed `C2paError` raised from the `Reader`/`Builder` call that triggered the fetch. An `except MyCustomError:` around `c2pa.Reader(...)` will never fire. Callers should catch `c2pa.C2paError` and read the message. - **Shape errors are caught early.** Returning a `str` body, a `None` or `bool` status, or a status outside the 100-599 range is rejected inside the trampoline with a clear `TypeError` message, which then surfaces the same way (as a `C2paError`). @@ -141,15 +131,27 @@ sequenceDiagram - **No reentrancy.** A custom resolver must not call c2pa APIs from inside `resolve()`. Re-entering the FFI while a call is in flight is undefined. - **Do not close while a call is in flight.** A `Context`/`Reader`/`Builder` must not be closed while a call is still running on that same object, including a resolver call it triggered. This is a general property of these objects, not something specific to resolvers. -## What the SDK leaves to the resolver +### A note on closures + +A resolver can be a plain function, a lambda, a bound method, or a `functools.partial`. It runs later on, when the resolver is needed/called. + +- **A closure captures the loop variable itself, not its value at creation time.** For instance, a resolver built inside a `for host in hosts:` loop shares one variable, `host`, across every iteration. By the time any resolver actually runs, the loop has finished and `host` holds whatever it was last set to, so every resolver built that way ends up using the same, final value. Giving the `lambda` its own `host` parameter, defaulted to `host`, fixes it. A default value is evaluated once, right when the `lambda` is created (on that specific loop iteration) — not later, when the `lambda` is finally called. So each `lambda` gets its own parameter holding a frozen copy of that iteration's `host`, shadowing the outer loop variable of the same name, instead of all of them reading the one shared loop variable after the loop has already finished: + + ```py + resolvers = [lambda req, host=host: fetch(host, req) for host in hosts] + ``` + +- **Capturing the `Context` (or a `Reader`/`Builder` made from it) creates a reference cycle that delays cleanup.** The SDK deliberately keeps the resolver alive for as long as the `Context` lives. If the resolver's closure also holds the `Context` (directly, or one step removed through a `Reader`/`Builder`, or via `functools.partial(fn, ctx)`), the two point at each other. Python's cyclic garbage collector still reclaims this — it is not a leak — but the native context handle is released later than expected, at an unpredictable time, instead of the moment the variable goes out of scope. Capturing only what the request actually needs (a session, a config), not the `Context`/`Reader`/`Builder` itself, avoids the cycle entirely: + + ```py + session = make_session() + ctx = c2pa.Context.builder().with_resolver(lambda r: helper(session, r)).build() + ``` -The SDK delegates the *transfer* entirely. The resolver *is* the HTTP client. That means: +- **The resolver, and anything it holds, stays alive for the whole `Context` lifetime — even without the application keeping its own reference to it.** With the `resolve()`-method form, the SDK holds the bound method, which holds its object. So an open HTTP session, connection pool, or background thread the resolver owns lives exactly as long as the `Context` does. It should not be relied on to be cleaned up early, and its teardown should not be tied to `Context.close()` — the resolver can still be invoked afterward (see above). Cleanup belongs to the resolver's own lifetime instead. +- **If an object is both callable and has a `resolve()` method, `resolve()` wins.** One shape should be picked: a plain function/lambda, or an object whose `resolve(request)` does the work, not something that is quietly both. -- **No redirects.** The SDK does not follow redirects. A `301`/`302` returned as-is is just a non-200. Delegating to `urllib.request` (as the network-facing examples do) gives redirect handling. Make sure to implement (or block) redirects as needed by the custom resolver. -- **Host filtering is bypassed.** The `core.allowed_network_hosts` setting only filters the *built-in* resolver. A custom resolver receives every request regardless. One that needs an allowlist must enforce it in `resolve()` (raise or return an error status for disallowed hosts). The URL comes from a remote-manifest reference embedded in the asset. The network-facing examples reject any URL whose scheme is not `http`/`https` before doing anything else. Validating the host (and, depending on the deployment, the resolved address and port) is worth considering too. -- **TLS belongs to the resolver.** Certificate verification, trust stores, and proxy handling all belong to whatever HTTP stack the custom resolver uses. The SDK sees only status and bytes. This is where most of the platform-specific behavior lives. See the platform section below. -- **No Content-Length plumbing.** The resolver response carries no `Content-Length`, so remote manifests larger than 10 MB are truncated **without an error**. A resolver that serves large manifests should note the failure mode downstream is a validation error, not a size error. -- **No caching, retries, or backoff.** Each needed resource is requested. Policy belongs to the custom resolver. `CachingHttpResolver` is the reference for a reasonable policy: cache only `GET`s answered with `200` (never `POST`s, since timestamp requests must not be replayed from cache, and never error responses), retry only `429`/`503` with a capped `Retry-After` or exponential backoff, and pass every other status through untouched. +The trampoline captures only the resolver function plus library internals, never the `Context`. It never creates a cycle on its own: the resolver is held by a strong reference, never a `weakref`, so it can't be garbage-collected mid-call. ## Platform differences: Linux, Windows, macOS @@ -182,7 +184,7 @@ flowchart TD E ==>|ownership transfers to native| F subgraph NA["Native side (SDK)"] F["copy the body out of the buffer"] --> G["free the buffer"] - G --> H["hand the bytes to the consumer:
manifest validation, OCSP, timestamp, or did:web"] + G --> H["hand the bytes to whatever needed the resource"] end ``` diff --git a/tests/http_resolver/http_resolver_example_impl.py b/tests/http_resolver/http_resolver_example_impl.py index 70924460..f51eaffa 100644 --- a/tests/http_resolver/http_resolver_example_impl.py +++ b/tests/http_resolver/http_resolver_example_impl.py @@ -22,6 +22,7 @@ the attribute shape the SDK expects. """ +import base64 import collections import threading import time @@ -40,6 +41,38 @@ def _reject_non_http(url): f"refusing to resolve non-http(s) URL scheme: {scheme!r}") +def _has_opaque_encoded_final_segment(url): + """True if the URL's last path segment looks like an encoded blob + (e.g. an OCSP request) rather than an ordinary resource name. + Such GETs should not be cached like a plain resource fetch. + """ + seg = urllib.parse.urlsplit(url).path.rstrip("/").rsplit("/", 1)[-1] + seg = urllib.parse.unquote(seg) + if len(seg) < 40: + return False + try: + base64.b64decode(seg, validate=True) + except Exception: + return False + return True + + +def _is_safe_to_cache(request): + """Oonly True for a GET positively known to be safe + (not opaque-encoded, not a DID document request). + Anything else (POSTs, ambiguous or unrecognized GETs) + is False by default. + """ + if request.method.upper() != "GET": + return False + if _has_opaque_encoded_final_segment(request.url): + return False + headers = {k.lower(): v for k, v in request.headers.items()} + if "application/did+json" in headers.get("accept", ""): + return False + return True + + class HttpRequest: """An HTTP request the SDK asks a custom resolver to perform. @@ -48,13 +81,14 @@ class HttpRequest: method: HTTP method ("GET", "POST", ...). headers: Request headers as a dict. Names are lowercased by the native layer; when a header repeats, the last value wins. - body: Request body bytes (b"" when there is none). Timestamp - requests POST a body; manifest fetches send none. + body: Request body bytes (b"" when there is none). A request + carrying a payload (typically a POST) sets this; a plain GET + usually doesn't. Reference shape only: - c2pa hands your resolve() a duck-typed object with these same four attributes, + c2pa hands resolve() a duck-typed object with these same four attributes, not necessarily an instance of this exact class. - Define your own compatible type, or just use this one. + A compatible type can be defined separately, or this one reused directly. """ __slots__ = ("url", "method", "headers", "body") @@ -73,8 +107,8 @@ class HttpResponse: """The answer a custom resolver returns to the SDK. Attributes: - status: HTTP status code. Remote manifest fetches only accept 200; - any other code surfaces as a typed C2paError. + status: HTTP status code. Only 200 is treated as success; any + other code surfaces as a typed C2paError. body: Response body bytes. Reference shape only: @@ -109,6 +143,12 @@ def resolve(self, request: HttpRequest) -> HttpResponse: raise NotImplementedError +# The resolvers implemented here are classes that hold their own config in +# __init__ and never capture a Context, Reader, or Builder in a closure. +# This avoids reference-cycles and lifetime issues. +# It also means resolve() never reaches for a mutable global. + + class TtlLruCache: """A small LRU cache whose entries also expire after a TTL.""" @@ -144,19 +184,18 @@ def put(self, key, value): class CachingHttpResolver(HttpResolver): """An HTTP resolver with a response cache and bounded retries. - Caching policy: - - Only read GET requests answered with 200 are cached. - - POSTs (timestamp requests) and error responses are not. + Caching policy: only GETs `_is_safe_to_cache` positively clears, + answered with 200, are cached. POSTs, errors, and anything not + positively cleared are never cached (see `_is_safe_to_cache`). - Retry policy: 429 and 503 are retried up to max_retries times, - honoring a capped Retry-After when the header is present, - and otherwise backing off exponentially. - Any other status is final and is passed through to the SDK. - Transport errors raise, which marks a hard failure. + Retry policy: 429/503 retried up to max_retries, honoring a capped + Retry-After or backing off exponentially. Any other status is final. + Transport errors raise, marking a hard failure. """ def __init__(self, cache=None, timeout=10.0, max_retries=3, backoff_seconds=0.5, max_retry_after=10.0): + # Capture config here self.cache = cache if cache is not None else TtlLruCache() self._timeout = timeout self._max_retries = int(max_retries) @@ -165,7 +204,7 @@ def __init__(self, cache=None, timeout=10.0, max_retries=3, def resolve(self, request): _reject_non_http(request.url) - cacheable = request.method.upper() == "GET" + cacheable = _is_safe_to_cache(request) if cacheable: cached = self.cache.get(request.url) if cached is not None: @@ -230,7 +269,6 @@ def resolve(self, request): _reject_non_http(request.url) self.requests.append((request.method, request.url)) - # Timestamp requests POST a body, manifest fetches send none. data = request.body or None req = urllib.request.Request( request.url, @@ -243,6 +281,7 @@ def resolve(self, request): return HttpResponse(resp.status, resp.read()) except urllib.error.HTTPError as e: # A 4xx/5xx is still a response. - # Pass the status through and let the SDK turn it into its own typed error: - # a remote manifest fetch only accepts 200 as marker the data was retrieved. + # Pass the status through and let the SDK turn it + # into its own typed error. + # Only 200 is treated as success. return HttpResponse(e.code, e.read()) diff --git a/tests/http_resolver/test_http_resolver_cache.py b/tests/http_resolver/test_http_resolver_cache.py index 9fee19f3..32b9c14b 100644 --- a/tests/http_resolver/test_http_resolver_cache.py +++ b/tests/http_resolver/test_http_resolver_cache.py @@ -27,11 +27,77 @@ FIXTURES = os.path.join(_REPO_ROOT, "tests", "fixtures") from http_resolver_example_impl import ( # noqa: E402 - AlwaysFailResolver, CachingHttpResolver) + AlwaysFailResolver, CachingHttpResolver, HttpRequest) import c2pa # noqa: E402 +class _StubFetchCachingResolver(CachingHttpResolver): + """CachingHttpResolver with the network swapped for a canned answer, + so the caching *decision* can be tested without a real HTTP call. + """ + + def __init__(self, status=200, body=b"payload"): + super().__init__() + self._canned = (status, body) + self.fetch_count = 0 + + def _fetch_with_retries(self, request): + self.fetch_count += 1 + return self._canned + + +class TestCachingDecision(unittest.TestCase): + """Tests for what CachingHttpResolver will and won't cache. + """ + + def test_opaque_encoded_get_is_never_cached(self): + """A GET whose URL ends in a long opaque token (the shape used by + e.g. a certificate revocation check) is fetched fresh every time, + never served from the cache. + """ + resolver = _StubFetchCachingResolver() + request = HttpRequest( + url="https://ca.example/ocsp/" + ("a" * 60), + method="GET", headers={}, body=b"") + + resolver.resolve(request) + resolver.resolve(request) + + self.assertEqual(resolver.fetch_count, 2) + self.assertEqual(resolver.cache.hits, 0) + + def test_post_is_never_cached(self): + """A request carrying a payload is never replayed from the cache.""" + resolver = _StubFetchCachingResolver() + request = HttpRequest( + url="https://tsa.example/timestamp", method="POST", + headers={"content-type": "application/timestamp-query"}, + body=b"query") + + resolver.resolve(request) + resolver.resolve(request) + + self.assertEqual(resolver.fetch_count, 2) + self.assertEqual(resolver.cache.hits, 0) + + def test_ordinary_get_is_cached(self): + """A plain GET the resolver can't positively rule out is treated + as an ordinary resource fetch and cached, so the fix doesn't + disable caching for the case it's meant to serve. + """ + resolver = _StubFetchCachingResolver() + request = HttpRequest( + url="https://cdn.example/manifests/x.c2pa", method="GET", + headers={}, body=b"") + + resolver.resolve(request) + resolver.resolve(request) + + self.assertEqual(resolver.fetch_count, 1) + self.assertEqual(resolver.cache.hits, 1) + + class TestHttpResolverCache(unittest.TestCase): def test_read_with_cache(self): @@ -154,6 +220,7 @@ def test_failing_resolver_is_reported_as_error(self): with self.assertRaises(c2pa.C2paError) as ctx: with open(os.path.join(FIXTURES, "cloud.jpg"), "rb") as f: c2pa.Reader("image/jpeg", f, context=context) + self.assertIn("500", str(ctx.exception)) if __name__ == "__main__": diff --git a/tests/http_resolver/test_http_resolver_debug.py b/tests/http_resolver/test_http_resolver_debug.py index 27da1e72..2618a4a4 100644 --- a/tests/http_resolver/test_http_resolver_debug.py +++ b/tests/http_resolver/test_http_resolver_debug.py @@ -11,7 +11,8 @@ # specific language governing permissions and limitations under # each license. -"""Tests for DebugHttpResolver, an HTTP resolver that logs requests, +"""Tests for DebugHttpResolver, an HTTP resolver that logs requests and +delegates the actual transfer to urllib. """ import json From f72879fd28e06ae50c82e8adf1c889749db77248 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:05:19 -0700 Subject: [PATCH 67/67] Update dependency description in http_resolver_example_impl.py Clarified the module's dependency statement regarding c2pa. --- tests/http_resolver/http_resolver_example_impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/http_resolver/http_resolver_example_impl.py b/tests/http_resolver/http_resolver_example_impl.py index f51eaffa..e91efda5 100644 --- a/tests/http_resolver/http_resolver_example_impl.py +++ b/tests/http_resolver/http_resolver_example_impl.py @@ -18,8 +18,8 @@ is never isinstance-checked, so any of these classes is a starting point to copy and adapt, not a required dependency. -This module has no dependency on c2pa itself: it only needs to satisfy -the attribute shape the SDK expects. +The implementation satisfies the object format/contract the SDK +expects when using custom resolvers. """ import base64