diff --git a/Include/Python.h b/Include/Python.h index 0b02377d51c776..f2591e9ec6e2f9 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -146,6 +146,7 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" #include "immutability.h" +#include "region.h" // Restore warning filter #ifdef _MSC_VER diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h index 7a7d37ce0dd07c..5f0dcc63b2a02c 100644 --- a/Include/internal/pycore_immutability.h +++ b/Include/internal/pycore_immutability.h @@ -9,13 +9,8 @@ extern "C" { #endif struct _Py_immutability_state { - PyObject *module_locks; - PyObject *blocking_on; PyObject *freezable_types; PyObject *destroy_cb; -#ifdef Py_DEBUG - PyObject *traceback_func; // For debugging purposes, can be NULL -#endif }; #ifdef __cplusplus diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e0d5ea69580e7e..789ee6a864b38c 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_immutability.h" // struct _immutability_runtime_state #include "pycore_llist.h" // struct llist_node #include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS +#include "pycore_ownership.h" // struct _Py_ownership_state #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl @@ -937,6 +938,7 @@ struct _is { struct _Py_exc_state exc_state; struct _Py_immutability_state immutability; struct _Py_mem_interp_free_queue mem_free_queue; + _Py_ownership_state ownership; struct ast_state ast; struct types_state types; diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 4289b2970f21aa..191500ba3ed693 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -77,7 +77,8 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ .ob_gc_bits = _PyGC_BITS_DEFERRED, \ - .ob_type = (type) \ + .ob_type = (type), \ + .ob_region = _Py_LOCAL_REGION \ } #else #if SIZEOF_VOID_P > 4 @@ -85,13 +86,15 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); { \ .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ .ob_flags = _Py_STATIC_FLAG_BITS, \ - .ob_type = (type) \ + .ob_type = (type), \ + .ob_region = _Py_LOCAL_REGION \ } #else #define _PyObject_HEAD_INIT(type) \ { \ .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \ - .ob_type = (type) \ + .ob_type = (type), \ + .ob_region = _Py_LOCAL_REGION \ } #endif #endif @@ -531,6 +534,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) Py_SET_TYPE(op, typeobj); assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); _Py_INCREF_TYPE(typeobj); + op->ob_region = _Py_LOCAL_REGION; _Py_NewReference(op); } diff --git a/Include/internal/pycore_ownership.h b/Include/internal/pycore_ownership.h new file mode 100644 index 00000000000000..2cbc9afc3fbe41 --- /dev/null +++ b/Include/internal/pycore_ownership.h @@ -0,0 +1,164 @@ +#ifndef Py_INTERNAL_OWNERSHIP_H +#define Py_INTERNAL_OWNERSHIP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#include "exports.h" +#include "object.h" + +typedef struct _Py_ownership_state { + /* The global ownership tick used to mark open regions as dirty, if their + * invariant might broken. This can happen if untrusted C code is called + * which doesn't have write barriers. This C code might create references + * between objects which could violate the invariant. Marking a region as + * dirty means that it has to be cleaned, before the region can be closed. + * + * The tick has two kinds of values: + * - Even => A region was opened + * - Odd => Untrusted code was called and all currently open regions + * should be marked as dirty. + * + * Transitions by increment: + * - From even to odd => Unknown C code was called + * - From odd to even => A new region was opened + * + * This mechanism allows marking all regions as dirty with a single tick + * change. + * + * Invariant: The tick counter should always be greater or equal to two + * as the values 0 and 1 are reserved values by `_Py_region_data.open_tick`. + * */ + Py_ssize_t tick; + // FIXME: xFrednet: Can we remove this special casing in favor of + // unfreezable fields or thread local wrappers. + PyObject *module_locks; + PyObject *blocking_on; +#ifdef Py_OWNERSHIP_INVARIANT + /* Tracks the state of the ownership invariant. Some ownership-related + * operations may temporarily violate the invariant. To handle this safely, + * the invariant must be suspended during such operations and only resumed + * once all of them complete. This is necessary to support re-entrancy. + * + * For example, during freezing, the object graph is traversed and objects + * are marked as immutable — even while they may still reference mutable + * objects. If the invariant were enforced mid-way, it would raise a + * (premature) error, despite the state being corrected as the operation + * completes. To avoid this, the invariant must be paused during the freeze. + * + * States: + * -1 => The invariant is disabled. + * 0 => The invariant is active and enforced. + * N => The invariant is temporarily paused. The value indicates the + * number of suspensions yet to be resumed (this supports nesting). + */ + int invariant_state; +#endif +#ifdef Py_DEBUG + /* Function to create a traceback object in debug builds. This is only used + * for debugging and can be NULL + */ + PyObject *traceback_func; + PyObject *location_key; +#endif +} _Py_ownership_state; + +/* This retrives the current ownership tick or 0 if the tick retrival failed. +* See `_Py_ownership_state.tick` +*/ +PyAPI_FUNC(Py_ssize_t) _PyOwnership_get_current_tick(void); + +/* Returns the tick which should be used for `region.open_tick` or 0 if the +* ownerstate is currently unavialble. +*/ +PyAPI_FUNC(Py_ssize_t) _PyOwnership_get_open_region_tick(void); + +/* This function should be called when, untrusted code is executed. It will +* mark all currently open regions as dirty. +* +* It can fail, if the ownership state is currently unavailable +*/ +PyAPI_FUNC(int) _PyOwnership_notify_untrusted_code(void); + + +PyAPI_FUNC(int) _PyOwnership_is_c_wrapper(PyObject *obj); + +/* Called for every object, to check what should be done with it. This + * can be used to implemented a set visited objects and avoid traversing + * objects multiple times. + * + * The return value indicates success and if the object should be + * traversed. These are the return values: + * -1) Failure + * 0) Ok, but don't traverse the object + * 1) Ok, and traverse the object + */ +typedef int (*ownershipcheckproc)(PyObject* obj, void *state); + +/* Like `visitproc` for `_PyOwnership_traverse_object_graph`. The first + * argument is the source of the reference and the second one is the + * referenced object. + * + * The return value indicates success and if the target object should be + * traversed. These are the return values: + * -1) Failure, stop traversal + * 0) Ok, but don't traverse the target object + * 1) Ok, and traverse the target object + */ +typedef int (*ownershipvisitproc)(PyObject* src, PyObject* tgt, void *state); + +#define Py_OWNERSHIP_TRAVERSE_ERR -1 +#define Py_OWNERSHIP_TRAVERSE_SKIP 0 +#define Py_OWNERSHIP_TRAVERSE_VISIT 1 + +PyAPI_FUNC(int) _PyOwnership_traverse_object_graph( + PyObject *obj, +#ifdef Py_DEBUG + int freeze_location, +#endif + ownershipcheckproc caller_check, + ownershipvisitproc caller_visit, + void *caller_state +); + +#ifdef Py_OWNERSHIP_INVARIANT + +#include "object.h" // PyObject, visitproc +#include "pytypedefs.h" // PyThreadState + +#define Py_OWNERSHIP_INVARIANT_DISABLED -1 +#define Py_OWNERSHIP_INVARIANT_ENABLED 0 + +/* This function validates that the current heap follows the ownership + * rules. This is a slow operation and should only be done for debugging. + * + * 0 indicates a valid heap, -1 will be returned if an error was thrown. + */ +PyAPI_FUNC(int) _PyOwnership_check_invariant(PyThreadState *tstate); + +PyAPI_FUNC(int) _PyOwnership_invariant_enable(void); +PyAPI_FUNC(int) _PyOwnership_invariant_pause(void); +PyAPI_FUNC(int) _PyOwnership_invariant_resume(void); +PyAPI_FUNC(int) _PyOwnership_invariant_disable(void); + +typedef struct _Py_ownership_invariant_region_data { + Py_region_t next; + Py_ssize_t lrc; + Py_ssize_t osc; +} _Py_ownership_invariant_region_data; + +#else +# define _PyOwnership_invariant_enable() 0 /* success */ +# define _PyOwnership_invariant_pause() 0 /* success */ +# define _PyOwnership_invariant_resume() 0 /* success */ +# define _PyOwnership_invariant_disable() 0 /* success */ +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OWNERSHIP_H */ diff --git a/Include/internal/pycore_region.h b/Include/internal/pycore_region.h new file mode 100644 index 00000000000000..706a65aa75bd5c --- /dev/null +++ b/Include/internal/pycore_region.h @@ -0,0 +1,146 @@ +#ifndef Py_INTERNAL_REGION_H +#define Py_INTERNAL_REGION_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#include "object.h" +#include "region.h" +#include "pycore_ownership.h" +#include "pycore_gc.h" // PyGC_Head + +/* Macros for readability */ +#define NULL_REGION 0 + +/* PyObject_HEAD defines the initial segment of every PyObject used as a region bridge. */ +#define PyBridgeObject_HEAD \ + PyObject_HEAD; \ + /* The region value which will be updated and \ + * still filled when the dealloc function of \ + * the object is called. \ + */ \ + Py_region_t region; \ + /** The name of the region or NULL */ \ + PyObject *name; + +#define PyBridgeObject_HEAD_INIT(op) \ + op->region = NULL_REGION; \ + op->name = NULL; + +/** + * Objects used as bridges need to have an additional region field, which is + * still filled in the dealloc function. This should be the inital segment, + * similar to how `PyObject` is the inital segment for other objects. +*/ +typedef struct _PyBridgeObject { + PyBridgeObject_HEAD; +} _PyBridgeObject; + +#define _PyBridgeObject_CAST(op) _Py_CAST(_PyBridgeObject*, op) + +typedef struct _Py_region_data { + /* The number of references coming in from the local region. + * + * This value should always be >= 0 with the exception of + * the `add_to_region` process. This can create a temporary + * region, which will be merged into the target region. The + * LRC can be negative, if the merge should decrease the LRC + * of the target region. + */ + Py_ssize_t lrc; + + /* The number of open subregions. */ + Py_ssize_t osc; + + /* Snapshot of the ownership tick, when the region was opened. This + * is used to track if the region is open and if the region is clean. + * + * If the region is clean, it means the LRC and OSC can be trusted to + * securely close the region. However, these values might be incorrect, + * if the region is dirty. This can happen, when we call untrusted C + * code. A dirty region first has to be cleaned, before it can be closed. + * + * See `_Py_ownership_state.tick` for an explaination of the tick counter. + * + * This value indicates the following states: + * - (0) => The region is closed + * - (1) => The region is open and dirty + * - (N) if N == state.tick => The region is open and clean, since the + * ownership and open tick are the same + * - (N) if N != state.tick => The region is open but dirty, since an + * ownership tick was triggered. + * + * Invariant: The open tick should always be 1 or an even number. + */ + Py_ssize_t open_tick; + + /* The number of references to this object */ + Py_ssize_t rc; + + /* A tagged pointer to the owner of this region. The tag indicates the + * type of owner and relationship: + * + * These are the possible tags: + * - 0b00 => The pointer points to the parent region (or is null) + * - 0b01 => The pointer points to the cown owing this region + * - 0b10 => The pointer points to the parent in the union-find forest + * - 0b11 => The pointer points to the parent in the union-fing forest, but the + * merge is not confirmed yet. Meaning references should not updated. + * + * Use the macros in `regions.c` to access these + */ + Py_uintptr_t owner; + + /* The bridge object belonging to this _Py_region_data. This pointer can be + * NULL, when the bridge was already deallocated but some objects retain + * a reference to the `_Py_region_data` object. + * + * This is a weak reference to the brige, meaning the RC is not updated + * by writes to this field. + */ + _PyBridgeObject* bridge; + + /* Objects have to be removed from their local GC cycle, when they're moved + * into a region. Instead they're moved into this list, to allow GC inside + * the region. + * + * Bridges can't form cycles with objects outside their regions (Mudolo cowns). + * It should therefore be safe to take them out of the GC cycle. + */ + PyGC_Head gc_list; + +#ifdef Py_OWNERSHIP_INVARIANT + _Py_ownership_invariant_region_data invariant_data; +#endif +} _Py_region_data; + +PyAPI_FUNC(int) _PyRegion_New(_PyBridgeObject *bridge); +PyAPI_FUNC(int) _PyRegion_Dissolve(Py_region_t region); +PyAPI_FUNC(void) _PyRegion_DecRc(Py_region_t region); + +PyAPI_FUNC(Py_ssize_t) _PyRegion_GetLrc(Py_region_t region); +PyAPI_FUNC(Py_ssize_t) _PyRegion_GetOsc(Py_region_t region); +PyAPI_FUNC(int) _PyRegion_IsOpen(Py_region_t region); +PyAPI_FUNC(int) _PyRegion_IsDirty(Py_region_t region); +PyAPI_FUNC(int) _PyRegion_IsParent(Py_region_t child, Py_region_t parent); +PyAPI_FUNC(int) _PyRegion_ClosesWithLrc(Py_region_t region, Py_ssize_t lrc); +PyAPI_FUNC(Py_region_t) _PyRegion_GetParent(Py_region_t child); +PyAPI_FUNC(int) _PyRegion_Clean(Py_region_t region); + +PyAPI_FUNC(int) _PyRegion_IsBridge(PyObject *obj); +PyAPI_FUNC(PyObject*) _PyRegion_GetBridge(Py_region_t region); +PyAPI_FUNC(void) _PyRegion_RemoveBridge(Py_region_t region); + +PyAPI_FUNC(int) _PyRegion_SignalImmutable(PyObject *obj); +PyAPI_FUNC(void) _PyRegion_SignalDealloc(PyObject *obj); + +PyAPI_FUNC(void) _PyRegion_HackDirtyForPrototype(Py_region_t region); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_REGION_H */ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 8791476725289c..110828db3e25bc 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_object.h" // Py_DECREF_MORTAL #include "pycore_object_deferred.h" // _PyObject_HasDeferredRefcount() +#include "region.h" // _PyRegion_RemoveLocalRef() #include // bool @@ -672,7 +673,9 @@ PyStackRef_CLOSE(_PyStackRef ref) { assert(!PyStackRef_IsNull(ref)); if (PyStackRef_RefcountOnObject(ref)) { - Py_DECREF_MORTAL(BITS_TO_PTR(ref)); + PyObject *ob = BITS_TO_PTR(ref); + _PyRegion_RemoveLocalRef(ob); + Py_DECREF_MORTAL(ob); } } #endif diff --git a/Include/object.h b/Include/object.h index 2a386c6d3e37cf..1c19e3a73fee7e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -56,6 +56,19 @@ whose size is determined when the object is allocated. # define Py_REF_DEBUG #endif +/* The identifier of a region. Functions in `pycore_regions.h` can be used to + * get metadata from this pointer. + */ +typedef Py_uintptr_t Py_region_t; + +/* A constant value used for the local region. Using a constant besides 0 leads + * to segementation falts, likely due to custom manual object initialization + * without `PyObject_HEAD_INIT`. + */ +#define _Py_LOCAL_REGION ((Py_region_t)0) +#define _Py_IMMUTABLE_REGION ((Py_region_t)4) +#define _Py_COWN_REGION ((Py_region_t)8) + /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -77,12 +90,14 @@ whose size is determined when the object is allocated. _Py_IMMORTAL_REFCNT_LOCAL, \ 0, \ (type), \ + (_Py_LOCAL_REGION) \ }, #else #define PyObject_HEAD_INIT(type) \ { \ { _Py_STATIC_IMMORTAL_INITIAL_REFCNT }, \ - (type) \ + (type), \ + (_Py_LOCAL_REGION) \ }, #endif @@ -142,6 +157,7 @@ struct _object { #endif PyTypeObject *ob_type; + Py_region_t ob_region; }; #else // Objects that are not owned by any thread use a thread id (tid) of zero. @@ -160,6 +176,7 @@ struct _object { uint32_t ob_ref_local; // local reference count Py_ssize_t ob_ref_shared; // shared (atomic) reference count PyTypeObject *ob_type; + Py_region_t ob_region; }; #endif diff --git a/Include/region.h b/Include/region.h new file mode 100644 index 00000000000000..a476c07d6311aa --- /dev/null +++ b/Include/region.h @@ -0,0 +1,74 @@ +#ifndef Py_REGION_H +#define Py_REGION_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "object.h" +#include "exports.h" + +typedef Py_uintptr_t PyRegion_staged_ref_t; +#define PyRegion_staged_ref_ERR 0 + +PyAPI_FUNC(Py_region_t) _PyRegion_GetSlow(PyObject *obj); + +/* Returns the region of the given object. + */ +static inline Py_region_t _PyRegion_Get(PyObject *obj) { + if (obj == NULL) { + return _Py_IMMUTABLE_REGION; + } + + // Immutable objects can be shared across threads, it's not safe to access + // the region information without synchronization. + if (_Py_IsImmutable(obj)) { + return _Py_IMMUTABLE_REGION; + } + + // Fast path, almost every object should be in one of these regions + if (obj->ob_region == _Py_LOCAL_REGION + || obj->ob_region == _Py_COWN_REGION + ) { + return obj->ob_region; + } + + return _PyRegion_GetSlow(obj); +} +#define _PyRegion_GET(obj) _PyRegion_Get(_PyObject_CAST(obj)) + +static inline int _Py_IsLocal(PyObject *obj) { + return _PyRegion_Get(obj) == _Py_LOCAL_REGION; +} +#define _Py_IsLocal(obj) _Py_IsLocal(_PyObject_CAST(obj)) + + +// Helper macros to count the number of arguments +#define _PyRegion__COUNT_ARGS(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, N, ...) N +#define _PyRegion_COUNT_ARGS(...) _PyRegion__COUNT_ARGS(__VA_ARGS__, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) +#define _PyRegion_MAX_ARG_COUNT 16 + +PyAPI_FUNC(PyRegion_staged_ref_t) _PyRegion_StageRef(PyObject *src, PyObject *tgt); +PyAPI_FUNC(void) _PyRegion_ResetStagedRef(PyRegion_staged_ref_t staged_ref); +PyAPI_FUNC(void) _PyRegion_CommitStagedRef(PyRegion_staged_ref_t staged_ref); +#define _PyRegion_STAGEREF(src, tgt) _PyRegion_StageRef(_PyObject_CAST(src), _PyObject_CAST(tgt)) + +PyAPI_FUNC(int) _PyRegion_AddRef(PyObject *src, PyObject *tgt); +PyAPI_FUNC(int) _PyRegion_AddRefs(PyObject *src, int tgt_count, ...); +#define _PyRegion_ADDREF(src, tgt) _PyRegion_AddRef(_PyObject_CAST(src), _PyObject_CAST(tgt)) +#define _PyRegion_ADDREFS(src, ...) _PyRegion_AddRefs(_PyObject_CAST(src), _PyRegion_COUNT_ARGS(__VA_ARGS__), __VA_ARGS__) + +PyAPI_FUNC(int) _PyRegion_RemoveRef(PyObject *src, PyObject *tgt); +#define _PyRegion_REMOVEREF(src, tgt) _PyRegion_RemoveRef(_PyObject_CAST(src), _PyObject_CAST(tgt)) + +PyAPI_FUNC(int) _PyRegion_AddLocalRef(PyObject *tgt); +PyAPI_FUNC(int) _PyRegion_AddLocalRefs(int tgt_count, ...); +#define _PyRegion_ADDLOCALREF(tgt) _PyRegion_AddLocalRef(_PyObject_CAST(tgt)) +#define _PyRegion_ADDLOCALREFS(...) _PyRegion_AddLocalRefs(_PyRegion_COUNT_ARGS(__VA_ARGS__), __VA_ARGS__) + +PyAPI_FUNC(int) _PyRegion_RemoveLocalRef(PyObject *tgt); +#define _PyRegion_REMOVELOCALREF(tgt) _PyRegion_RemoveLocalRef(_PyObject_CAST(tgt)) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_REGION_H */ \ No newline at end of file diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index b270024dd7df98..e2ed8ef5714fd6 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -465,15 +465,6 @@ def test_weakref(self): # self.assertTrue(c.val() is obj) self.assertIsNone(c.val()) -class TestStackCapture(unittest.TestCase): - def test_stack_capture(self): - import sys - x = {} - x["frame"] = sys._getframe() - freeze(x) - self.assertTrue(isfrozen(x)) - self.assertTrue(isfrozen(x["frame"])) - global_test_dict = 0 class TestGlobalDictMutation(unittest.TestCase): def g(): diff --git a/Lib/test/test_freeze/test_gc.py b/Lib/test/test_freeze/test_gc.py index 8c20d9af7585b8..23a93f0ead46d4 100644 --- a/Lib/test/test_freeze/test_gc.py +++ b/Lib/test/test_freeze/test_gc.py @@ -1,6 +1,6 @@ from gc import collect import unittest -from immutable import freeze, NotFreezable, isfrozen +from immutable import freeze class GCInteropTest(unittest.TestCase): def test_collect(self): @@ -11,4 +11,4 @@ def test_collect(self): # Freeze it freeze(a) # f - collect() \ No newline at end of file + collect() diff --git a/Lib/test/test_regions/__init__.py b/Lib/test/test_regions/__init__.py new file mode 100644 index 00000000000000..ca273763bed98d --- /dev/null +++ b/Lib/test/test_regions/__init__.py @@ -0,0 +1,6 @@ +import os +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_regions/__main__.py b/Lib/test/test_regions/__main__.py new file mode 100644 index 00000000000000..20c011aed4c548 --- /dev/null +++ b/Lib/test/test_regions/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main() diff --git a/Lib/test/test_regions/test_clean.py b/Lib/test/test_regions/test_clean.py new file mode 100644 index 00000000000000..8b237e05aa8e03 --- /dev/null +++ b/Lib/test/test_regions/test_clean.py @@ -0,0 +1,59 @@ +import unittest +from regions import Region, is_local +import sys + + +class TestCleanRegion(unittest.TestCase): + def mark_region_as_dirty(self, region: Region): + # FIXME(regions): xFrednet: Currently all regions are marked as dirty + # while most write barriers are missing. This will later need some + # magic to mark the region as dirty + self.assertTrue(region.is_dirty, "Region should be dirty here") + + def test_try_close_dirty_with_local_ref(self): + region = Region() + + self.mark_region_as_dirty(region) + region.clean() + + self.assertFalse(region.is_dirty) + + def test_try_close_sub_region(self): + region = Region() + region.sub = Region() + self.mark_region_as_dirty(region) + self.mark_region_as_dirty(region.sub) + + sub = region.sub + + # Cleaning a dirty parent region should clean the child as well + region.clean() + + # The region should now be clean + self.assertFalse(sub.is_dirty) + + def test_clean_removes_unreachable(self): + region = Region() + obj = {} + region.x = obj + region.x = None + + # `region` should remain the owner of obj + self.assertTrue(region.owns(obj)) + + # Make the region dirty and clean it + self.mark_region_as_dirty(region) + region.clean() + + # Clean should have kicked `obj` from the region since it is no + # longer reachable from the bridge object + self.assertFalse(region.owns(obj)) + self.assertTrue(is_local(obj)) + + def test_clean_keeps_name(self): + region = Region("Marlin") + + self.mark_region_as_dirty(region) + region.clean() + + self.assertEqual(region.name, "Marlin") diff --git a/Lib/test/test_regions/test_core.py b/Lib/test/test_regions/test_core.py new file mode 100644 index 00000000000000..af88a125e806c3 --- /dev/null +++ b/Lib/test/test_regions/test_core.py @@ -0,0 +1,176 @@ +import unittest +from regions import Region, is_local +from immutable import freeze, isfrozen + +class TestBasicRegionObject(unittest.TestCase): + def test_region_construction(self): + r = Region() + + # A region should own itself + self.assertTrue(r.owns(r)) + + # A region should own its dict + self.assertTrue(r.owns(r.__dict__)) + + # The region should be open since r points into it + self.assertTrue(r.is_open) + + # FIXME(regions): xFrednet: Regions currently default to being dirty + # while most write barriers are missing + # + # A new region should be clean + # self.assertFalse(r.is_dirty) + + # A new region has no parent + self.assertIsNone(r.parent) + + def test_fields_read_only(self): + r = Region() + + # Check the exception on assignment + with self.assertRaises(AttributeError): + r.is_open = False + + with self.assertRaises(AttributeError): + r.is_dirty = False + + with self.assertRaises(AttributeError): + r.parent = None + +class ImplicitFreezingForImmortal(unittest.TestCase): + def test_implicit_freeze_importal(self): + # This would ideally check that the immortal objects + # are unfrozen, before we add them to a region. However, + # this would create an ordering dependency between tests. + # So here we just check that they're frozen after the fact. + r = Region() + + r.true = True + self.assertFalse(r.owns(r.true)) + self.assertTrue(isfrozen(r.true)) + self.assertEqual(r.true, True) + + r.num = 12 + self.assertFalse(r.owns(r.num)) + self.assertTrue(isfrozen(r.num)) + self.assertEqual(r.num, 12) + + r.none = None + self.assertFalse(r.owns(r.none)) + self.assertTrue(isfrozen(r.none)) + self.assertEqual(r.none, None) + +class TestOwnership(unittest.TestCase): + class A: + pass + + def setUp(self): + # Allows the A type to be referenced from multiple regions + freeze(self.A) + + def test_local_not_owned(self): + # Create a region + r = Region() + + # Create a new local object + a = self.A() + + self.assertTrue(is_local(a)) + self.assertFalse(r.owns(a)) + + def test_region_takes_ownership_of_local(self): + # Create a region + r = Region() + + # Create a new local object + a = self.A() + self.assertTrue(is_local(a)) + + # Move a into r + r.a = a + self.assertTrue(r.owns(a)) + self.assertFalse(is_local(a)) + + def test_region_takes_ownership_of_local_is_deep(self): + # Create a region + r = Region() + + # Create a new local object + a = self.A() + a.b = self.A() + self.assertTrue(is_local(a)) + self.assertTrue(is_local(a.b)) + + # Move a into r + r.a = a + self.assertTrue(r.owns(a)) + self.assertTrue(r.owns(a.b)) + self.assertFalse(is_local(a)) + self.assertFalse(is_local(a.b)) + +class TestInterRegionRelations(unittest.TestCase): + class A: + pass + + def setUp(self): + # Allows the A type to be referenced from multiple regions + freeze(self.A) + + def test_reference_to_contained(self): + r1 = Region() + r2 = Region() + a = self.A() + + # Move a into r1 + r1.a = a + self.assertTrue(r1.owns(a)) + self.assertFalse(r2.owns(a)) + + # Check the exception on assignment + with self.assertRaises(RuntimeError) as e: + r2.a = a + self.assertEqual(e.exception.source, r2.__dict__) + self.assertEqual(e.exception.target, a) + + # Check ownership is unchanged + self.assertTrue(r1.owns(a)) + self.assertFalse(r2.owns(a)) + + def test_unchanged_region_after_failure(self): + r1 = Region() + r2 = Region() + a = self.A() + a.b = self.A() + a.b.c = self.A() + + # Move a.b.c into r1 + r1.c = a.b.c + self.assertTrue(is_local(a)) + self.assertTrue(is_local(a.b)) + self.assertTrue(r1.owns(a.b.c)) + + # Moving a into r2 will fail due to a.b.c being in a different region + with self.assertRaises(RuntimeError) as e: + r2.a = a + self.assertEqual(e.exception.source, a.b) + self.assertEqual(e.exception.target, a.b.c) + + # Object a and b should remain local + self.assertTrue(is_local(a)) + self.assertTrue(is_local(a.b)) + + def test_get_parent(self): + r1 = Region() + r2 = Region() + + # Make r2 a child of r1 + r1.r2 = r2 + + # Check that r2 knows ab out this + self.assertEqual(r2.parent, r1) + + # Unparent r2 again + r1.r2 = None + + # Check that r2 has no parent + self.assertIsNone(r2.parent) diff --git a/Lib/test/test_regions/test_gc.py b/Lib/test/test_regions/test_gc.py new file mode 100644 index 00000000000000..3b3a3739bea0ba --- /dev/null +++ b/Lib/test/test_regions/test_gc.py @@ -0,0 +1,32 @@ +import unittest +from regions import Region, is_local +import gc + +class TestOwnership(unittest.TestCase): + class A: + pass + + def build_cycle(self): + a = self.A() + a.b = self.A() + a.b.a = a + return a + + def test_owned_cycles_are_ignored(self): + r = Region() + + # Make sure that there are no lingering cycles + gc.collect() + + # A normal cycle should be collected + self.build_cycle() + self.assertEqual(gc.collect(), 2) + + # A cycle inside a region should be ignored + r.c = self.build_cycle() + r.c = None + self.assertEqual(gc.collect(), 0) + + # Dissolving a region should allow cycles to be collected again + r = None + self.assertEqual(gc.collect(), 2) \ No newline at end of file diff --git a/Makefile.pre.in b/Makefile.pre.in index bf9ab195cecff9..f04f0e1bc293f6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -474,6 +474,7 @@ PYTHON_OBJS= \ Python/optimizer.o \ Python/optimizer_analysis.o \ Python/optimizer_symbols.o \ + Python/ownership.o \ Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ @@ -508,6 +509,7 @@ PYTHON_OBJS= \ Python/perf_trampoline.o \ Python/perf_jit_trampoline.o \ Python/remote_debugging.o \ + Python/region.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ @@ -1356,6 +1358,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Include/internal/pycore_ownership.h \ $(srcdir)/Include/internal/pycore_parking_lot.h \ $(srcdir)/Include/internal/pycore_parser.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ @@ -1374,6 +1377,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_qsbr.h \ $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_region.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ @@ -2649,6 +2653,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_peg_generator \ test/test_pydoc \ test/test_pyrepl \ + test/test_regions \ test/test_string \ test/test_sqlite3 \ test/test_tkinter \ diff --git a/Modules/Setup b/Modules/Setup index 53996b7bbcd61b..39076c4a0515bd 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -158,6 +158,7 @@ PYTHONPATH=$(COREPYTHONPATH) #cmath cmathmodule.c #math mathmodule.c #mmap mmapmodule.c +#regions regionsmodule.c #select selectmodule.c #_sysconfig _sysconfig.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 96062a8ab29525..aea35bf114b580 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -42,6 +42,7 @@ @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c +@MODULE_REGIONS_TRUE@regions regionsmodule.c @MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging_module.c @MODULE__STRUCT_TRUE@_struct _struct.c diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 93be44ea274477..c269c116f47917 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4467,7 +4467,7 @@ module_exec(PyObject *m) CREATE_TYPE(m, st->Element_Type, &element_spec); CREATE_TYPE(m, st->XMLParser_Type, &xmlparser_spec); - if (_PyImmutability_RegisterFreezable((PyObject *)st->Element_Type) != 0) { + if (_PyImmutability_RegisterFreezable((PyTypeObject *)st->Element_Type) != 0) { goto error; } diff --git a/Modules/clinic/regionsmodule.c.h b/Modules/clinic/regionsmodule.c.h new file mode 100644 index 00000000000000..bb4a7e6791a9da --- /dev/null +++ b/Modules/clinic/regionsmodule.c.h @@ -0,0 +1,32 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(regions_is_local__doc__, +"is_local($module, obj, /)\n" +"--\n" +"\n" +"Return True the object is in the local region."); + +#define REGIONS_IS_LOCAL_METHODDEF \ + {"is_local", (PyCFunction)regions_is_local, METH_O, regions_is_local__doc__}, + +static int +regions_is_local_impl(PyObject *module, PyObject *obj); + +static PyObject * +regions_is_local(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = regions_is_local_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=4d408aceaaaa05ff input=a9049054013a1b77]*/ diff --git a/Modules/regionsmodule.c b/Modules/regionsmodule.c new file mode 100644 index 00000000000000..32d0c539028ab8 --- /dev/null +++ b/Modules/regionsmodule.c @@ -0,0 +1,478 @@ +/* regions module */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#define MODULE_VERSION "1.0" + +#include "Python.h" +#include +#include "pycore_object.h" +#include "pycore_region.h" +#include "pycore_ownership.h" + +/*[clinic input] +module regions +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=38ff706d605d1871]*/ + +#include "clinic/regionsmodule.c.h" + +/* + * =================== + * Module State + * =================== + */ + +typedef struct regions_state { + PyObject *region_error_obj; +} regions_state; + +static struct PyModuleDef regionsmodule; + +static inline regions_state* +get_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (regions_state *)state; +} + +static int +regions_clear(PyObject *module) +{ + regions_state *module_state = get_state(module); + Py_CLEAR(module_state->region_error_obj); + return 0; +} + +static int +regions_traverse(PyObject *module, visitproc visit, void *arg) +{ + regions_state *module_state = get_state(module); + Py_VISIT(module_state->region_error_obj); + return 0; +} + +static void +regions_free(void *module) +{ + regions_clear((PyObject *)module); +} + +/* + * =================== + * RegionError + * =================== + */ + +static PyType_Slot region_error_slots[] = { + {0, NULL}, +}; + +PyType_Spec regions_error_spec = { + .name = "regions.RegionError", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = region_error_slots, +}; + +void RegionErr_NoBridge(void) { + // FIXME Static RegionError and call + PyErr_Format( + PyExc_RuntimeError, + "a region method was called on a non-bridge object"); +} + +/* + * =================== + * Region Object + * =================== + */ + +PyDoc_STRVAR(Region_doc, "FIXME =^.^="); + +typedef struct RegionObject { + PyBridgeObject_HEAD + PyObject *dict; +} RegionObject; + +#define RegionObject_CAST(op) ((RegionObject *)(op)) + +static PyMemberDef Region_members[] = { + {"__dict__", _Py_T_OBJECT, offsetof(RegionObject, dict), Py_READONLY}, + {NULL} +}; + +static int Region_init(RegionObject *self, PyObject *args, PyObject *kwds) { + // Parse optional parameter + static char *kwlist[] = {"name", NULL}; + PyObject *name = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &name)) { + return -1; + } + + // Strings are often interned, which makes sharing complicated. But + // they are effectivly immutable, which makes freezing a simple and + // safe fix. + if (name && _PyImmutability_Freeze(name)) { + return -1; + } + + PyBridgeObject_HEAD_INIT(self); + + // Allocate the new region object + if (_PyRegion_New(_PyBridgeObject_CAST(self))) { + return -1; + } + assert(self->region != NULL_REGION); + + // Check the object is also correctly moved into the region + assert(_PyRegion_Get(_PyObject_CAST(self)) == self->region); + assert(_PyRegion_IsBridge(_PyObject_CAST(self))); + + // No write barrier needed, since name is frozen + self->name = _Py_XNewRef(name); + + // Everything is a-okay + return 0; +} + +static PyObject * +Region_repr(PyObject *op) +{ + if (!_PyRegion_IsBridge(op)) { + return PyUnicode_FromString("");; + } + + RegionObject *self = RegionObject_CAST(op); + Py_region_t region = _PyRegion_Get(op); + + PyObject *repr = NULL; +#ifdef Py_DEBUG + repr = PyUnicode_FromFormat( + "", + self->name, + _PyRegion_GetLrc(region), + _PyRegion_GetOsc(region), + _PyRegion_IsDirty(region) ? "True" : "False" + ); +#else + repr = PyUnicode_FromFormat( + "", + self->name, + ); +#endif + + return repr; +} + +#define CHECK_BRIDGE(self) \ + if (!_PyRegion_IsBridge(_PyObject_CAST(self))) { \ + RegionErr_NoBridge(); \ + return NULL; \ + } + +static PyObject* Region_owns(PyObject *self, PyObject *other) { + CHECK_BRIDGE(self); + + Py_region_t self_region = _PyRegion_Get(self); + Py_region_t other_region = _PyRegion_Get(other); + return PyBool_FromLong(self_region == other_region); +} + +static PyObject* Region_clean(PyObject *op) { + CHECK_BRIDGE(op); + + if (_PyRegion_Clean(_PyRegion_Get(op))) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyMethodDef Region_methods[] = { + {"owns", _PyCFunction_CAST(Region_owns), METH_O, + "Check if object is owned by the region."}, + {"clean", _PyCFunction_CAST(Region_clean), METH_NOARGS, + "Cleans the region and any dirty subregions"}, + {NULL, NULL} /* sentinel */ +}; + +static PyObject* Region_is_open(PyObject *self, void *closure) { + CHECK_BRIDGE(self); + + int is_open = _PyRegion_IsOpen(_PyRegion_Get(self)); + return PyBool_FromLong(is_open); +} + +static PyObject* Region_is_dirty(PyObject *self, void *closure) { + CHECK_BRIDGE(self); + + int is_dirty = _PyRegion_IsDirty(_PyRegion_Get(self)); + return PyBool_FromLong(is_dirty); +} + +static PyObject* Region_get_parent(PyObject *self, void *closure) { + CHECK_BRIDGE(self); + + Py_region_t parent_region = _PyRegion_GetParent(_PyRegion_Get(self)); + return _Py_NewRef(_PyRegion_GetBridge(parent_region)); +} + +static PyObject* Region_get_name(PyObject *self, void *closure) { + CHECK_BRIDGE(self); + + return Py_NewRef(RegionObject_CAST(self)->name); +} + +static PyObject* Region_get__lrc(PyObject* self, void* closure) { + CHECK_BRIDGE(self); + + Py_ssize_t lrc = _PyRegion_GetLrc(_PyRegion_Get(self)); + return PyLong_FromSize_t(lrc); +} + +static PyObject* Region_get__osc(PyObject* self, void* closure) { + CHECK_BRIDGE(self); + + Py_ssize_t osc = _PyRegion_GetOsc(_PyRegion_Get(self)); + return PyLong_FromSize_t(osc); +} + +static PyGetSetDef Region_getset[] = { + {"is_open", (getter)Region_is_open, NULL, + "indicates if the region is currently open or closed", NULL}, + {"is_dirty", (getter)Region_is_dirty, NULL, + "indicates if the region is currently dirty", NULL}, + {"parent", (getter)Region_get_parent, NULL, + "the parent of the region", NULL}, + {"name", (getter)Region_get_name, NULL, + "the name of the region", NULL}, + {"_lrc", (getter)Region_get__lrc, NULL, + "the local-reference count, mainly intended for debugging", NULL}, + {"_osc", (getter)Region_get__osc, NULL, + "the open-subregion count, mainly intended for debugging", NULL}, + {NULL, NULL, NULL, NULL, NULL} +}; + +static int +Region_traverse(PyObject *op, visitproc visit, void *arg) +{ + RegionObject *self = RegionObject_CAST(op); + + // Visit the type + Py_VISIT(Py_TYPE(op)); + + // Only visit the name from the root bridge object + if (_PyRegion_IsBridge(op)) { + Py_VISIT(self->name); + } + + // Visit the attribute dict + Py_VISIT(self->dict); + return 0; +} + +static int +Region_clear(PyObject *op) +{ + RegionObject *self = RegionObject_CAST(op); + + if (self->region != NULL_REGION) { + // This merges this region into the local region. This is done because: + // (1) Once the bridge is gone, there is no way to send the region + // anymore therefore there is no advantage of tracking ownership + // for these objects + // (2) Clear might propagate through the object graph. This previously + // caused some asserts to fail, which assumed the bridge to always + // be there. + // (3) Only guessing, but merging the region back into the local region + // will probably be good for usability, since there is more freedom + // to reference previously contained objects. + _PyRegion_Dissolve(self->region); + + // Clear the region, this uses the internal region pointer + // since `_PyRegion_Get` might be different or already cleared. + _PyRegion_RemoveBridge(self->region); + _PyRegion_DecRc(self->region); + self->region = NULL_REGION; + } + + // Clear members + Py_CLEAR(self->name); + Py_CLEAR(self->dict); + return 0; +} + +static void +Region_dealloc(PyObject *self) +{ + // The region in the `ob_region` field should be cleared before calling + // dealloc. + assert(self->ob_region == NULL_REGION); + + PyObject_GC_UnTrack(self); + + Region_clear(self); + + PyTypeObject *tp = Py_TYPE(self); + freefunc free = PyType_GetSlot(tp, Py_tp_free); + free(self); + Py_DECREF(tp); +} + +/* The region type is intentionally static and immutable to allow save sharing + * across subinterpreters. Declaring it as static allows type comparisons to + * work automatically. + * + * One downside is, that the normal `PyType_GetModuleState` function doesn't + * work for static types. So everthing needs to either use static types or + * look up the `regions` module dynamically at runtime. + */ +static PyTypeObject Region_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "regions.Region", + .tp_basicsize = sizeof(RegionObject), + // .tp_itemsize = 0, + .tp_dealloc = (destructor)Region_dealloc, + .tp_getset = Region_getset, + // .tp_vectorcall_offset = 0, + // .tp_getattr = 0, + // .tp_setattr = 0, + // .tp_as_async = 0, + .tp_repr = (reprfunc)Region_repr, + // .tp_as_number = 0, + // .tp_as_sequence = 0, + // .tp_as_mapping = 0, + // .tp_hash = 0, + // .tp_call = 0, + // .tp_str = 0, + // .tp_getattro = 0, + // .tp_setattro = 0, + // .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .tp_doc = Region_doc, + .tp_traverse = (traverseproc)Region_traverse, + .tp_clear = (inquiry)Region_clear, + // .tp_richcompare = 0, + // .tp_weaklistoffset = 0, + // .tp_iter = 0, + // .tp_iternext = 0, + .tp_methods = Region_methods, + .tp_members = Region_members, + // .tp_getset = 0, + // .tp_base = 0, + // .tp_descr_get = 0, + // .tp_descr_set = 0, + .tp_dictoffset = offsetof(RegionObject, dict), + .tp_init = (initproc)Region_init, + // .tp_alloc = 0, + .tp_new = PyType_GenericNew, +}; + +/* + * =================== + * MODULE + * =================== + */ + +PyDoc_STRVAR(regions_module_doc, ""); + +/*[clinic input] +regions.is_local -> bool + obj: object + / + +Return True the object is in the local region. +[clinic start generated code]*/ + +static int +regions_is_local_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=e113b6b045da92b4 input=17b3dedc5693f308]*/ +{ + return _Py_IsLocal(obj); +} + +static struct PyMethodDef regions_methods[] = { + REGIONS_IS_LOCAL_METHODDEF + { NULL, NULL } +}; + + +static int +regions_exec(PyObject *module) { + regions_state *module_state = get_state(module); + + /* Add version to the module. */ + if (PyModule_AddStringConstant(module, "__version__", + MODULE_VERSION) == -1) { + return -1; + } + + // Create the `RegionError` type + PyObject *bases = PyTuple_Pack(1, PyExc_TypeError); + if (bases == NULL) { + return -1; + } + module_state->region_error_obj = PyType_FromModuleAndSpec( + module, + ®ions_error_spec, + bases); + Py_DECREF(bases); + if (module_state->region_error_obj == NULL) { + return -1; + } + if (PyModule_AddType(module, (PyTypeObject *)module_state->region_error_obj) != 0) { + return -1; + } + + // Register the `Region` type + if (PyType_Ready(&Region_Type) < 0) { + return -1; + } + if (_PyImmutability_Freeze(_PyObject_CAST(&Region_Type)) != 0) { + return -1; + } + _Py_SetImmortalUntracked(_PyObject_CAST(&Region_Type)); + if (PyModule_AddObject(module, "Region", _PyObject_CAST(&Region_Type)) < 0) { + return -1; + } + + // Freeze the dict type, to allow dictionaries to be used across regions. + if (_PyImmutability_Freeze(_PyObject_CAST(&PyDict_Type)) != 0) { + return -1; + } + + // Disable the invariant again, since it slows Python down so much + if (_PyOwnership_invariant_disable() != 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot regions_slots[] = { + {Py_mod_exec, regions_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_USED}, + {0, NULL} +}; + +static struct PyModuleDef regionsmodule = { + PyModuleDef_HEAD_INIT, + "regions", + regions_module_doc, + sizeof(regions_state), + regions_methods, + regions_slots, + regions_traverse, + regions_clear, + regions_free +}; + +PyMODINIT_FUNC +PyInit_regions(void) +{ + return PyModuleDef_Init(®ionsmodule); +} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 9bb1a18c9cbf6c..b6106fd2274804 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -131,6 +131,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_Recycle() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() +#include "region.h" // _PyRegion_ADDREFS #include "stringlib/eq.h" // unicode_eq() #include @@ -444,7 +445,7 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) +dictkeys_decref(PyInterpreterState *interp, PyObject *dict, PyDictKeysObject *dk, bool use_qsbr) { if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); @@ -459,6 +460,8 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); Py_ssize_t i, n; for (i = 0, n = dk->dk_nentries; i < n; i++) { + _PyRegion_RemoveRef(dict, entries[i].me_key); + _PyRegion_RemoveRef(dict, entries[i].me_value); Py_XDECREF(entries[i].me_key); Py_XDECREF(entries[i].me_value); } @@ -467,6 +470,8 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) PyDictKeyEntry *entries = DK_ENTRIES(dk); Py_ssize_t i, n; for (i = 0, n = dk->dk_nentries; i < n; i++) { + _PyRegion_RemoveRef(dict, entries[i].me_key); + _PyRegion_RemoveRef(dict, entries[i].me_value); Py_XDECREF(entries[i].me_key); Py_XDECREF(entries[i].me_value); } @@ -876,7 +881,7 @@ new_dict(PyInterpreterState *interp, if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys, false); + dictkeys_decref(interp, NULL, keys, false); if (free_values_on_failure) { free_values(values, false); } @@ -1801,6 +1806,7 @@ insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, else { _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + _PyRegion_REMOVEREF(mp, old_value); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_DECREF(old_value); @@ -1834,6 +1840,9 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (_PyDict_HasSplitTable(mp)) { + if (_PyRegion_ADDREF(mp, key) != 0) { + goto Fail; + } Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { insert_split_value(interp, mp, key, value, ix); @@ -1856,6 +1865,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, assert(!_PyDict_HasSplitTable(mp)); /* Insert into new slot. */ assert(old_value == NULL); + // Write Barrier called by `insert_combined_dict` if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } @@ -1865,6 +1875,10 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (old_value != value) { + if (_PyRegion_ADDREFS(mp, value) != 0) { + goto Fail; + } + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); assert(!_PyDict_HasSplitTable(mp)); @@ -1877,6 +1891,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, STORE_VALUE(ep, value); } } + _PyRegion_REMOVEREF(mp, old_value); Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ ASSERT_CONSISTENT(mp); Py_DECREF(key); @@ -1912,6 +1927,14 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } + + // Regions Write Barrier + if (_PyRegion_ADDREFS(mp, key, value) != 0) { + Py_DECREF(key); + Py_DECREF(value); + return -1; + } + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ @@ -1977,11 +2000,12 @@ build_indices_unicode(PyDictKeysObject *keys, PyDictUnicodeEntry *ep, Py_ssize_t } static void -invalidate_and_clear_inline_values(PyDictValues *values) +invalidate_and_clear_inline_values(PyObject *dict, PyDictValues *values) { assert(values->embedded); FT_ATOMIC_STORE_UINT8(values->valid, 0); for (int i = 0; i < values->capacity; i++) { + _PyRegion_RemoveRef(dict, values->values[i]); FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL); } } @@ -2072,12 +2096,12 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } UNLOCK_KEYS(oldkeys); set_keys(mp, newkeys); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, _PyObject_CAST(mp), oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); if (oldvalues->embedded) { assert(oldvalues->embedded == 1); assert(oldvalues->valid == 1); - invalidate_and_clear_inline_values(oldvalues); + invalidate_and_clear_inline_values(_PyObject_CAST(mp), oldvalues); } else { free_values(oldvalues, IS_DICT_SHARED(mp)); @@ -2395,6 +2419,9 @@ _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, Py #ifdef Py_GIL_DISABLED *result = value; #else + if (_PyRegion_AddLocalRef(value)) { + return -1; + } *result = Py_NewRef(value); #endif return 1; // key is present @@ -2641,10 +2668,32 @@ setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) int _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) { - int res; + int res = -1; + + // Check if the new references can be created + PyRegion_staged_ref_t staged_key = PyRegion_staged_ref_ERR; + PyRegion_staged_ref_t staged_value = PyRegion_staged_ref_ERR; + staged_key = _PyRegion_STAGEREF(mp, key); + staged_value = _PyRegion_STAGEREF(mp, value); + if (staged_key == PyRegion_staged_ref_ERR || staged_value == PyRegion_staged_ref_ERR) { + goto Fail; + } + + // Insert the value if possible Py_BEGIN_CRITICAL_SECTION(mp); res = setitem_take2_lock_held(mp, key, value); Py_END_CRITICAL_SECTION(); + if (res != 0) { + goto Fail; + } + + _PyRegion_CommitStagedRef(staged_key); + _PyRegion_CommitStagedRef(staged_value); + return 0; + +Fail: + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); return res; } @@ -2663,8 +2712,33 @@ PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) } assert(key); assert(value); - return _PyDict_SetItem_Take2((PyDictObject *)op, + + int res = -1; + + // Check if the new references can be created + PyRegion_staged_ref_t staged_key = PyRegion_staged_ref_ERR; + PyRegion_staged_ref_t staged_value = PyRegion_staged_ref_ERR; + staged_key = _PyRegion_STAGEREF(op, key); + staged_value = _PyRegion_STAGEREF(op, value); + if (staged_key == PyRegion_staged_ref_ERR || staged_value == PyRegion_staged_ref_ERR) { + goto Fail; + } + + res = _PyDict_SetItem_Take2((PyDictObject *)op, Py_NewRef(key), Py_NewRef(value)); + + if (res != 0) { + goto Fail; + } + + _PyRegion_CommitStagedRef(staged_key); + _PyRegion_CommitStagedRef(staged_value); + return 0; + +Fail: + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); + return res; } static int @@ -2672,8 +2746,29 @@ setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { assert(key); assert(value); - return setitem_take2_lock_held(mp, - Py_NewRef(key), Py_NewRef(value)); + + // Check if the new references can be created + PyRegion_staged_ref_t staged_key = PyRegion_staged_ref_ERR; + PyRegion_staged_ref_t staged_value = PyRegion_staged_ref_ERR; + staged_key = _PyRegion_STAGEREF(mp, key); + staged_value = _PyRegion_STAGEREF(mp, value); + if (staged_key == PyRegion_staged_ref_ERR || staged_value == PyRegion_staged_ref_ERR) { + goto Fail; + } + + // Insert the value if possible + if (setitem_take2_lock_held(mp, Py_NewRef(key), Py_NewRef(value))) { + goto Fail; + } + + _PyRegion_CommitStagedRef(staged_key); + _PyRegion_CommitStagedRef(staged_value); + return 0; + +Fail: + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); + return -1; } @@ -2682,11 +2777,35 @@ _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *va Py_hash_t hash) { PyInterpreterState *interp = _PyInterpreterState_GET(); + int res = -1; + + // Check if the new references can be created + PyRegion_staged_ref_t staged_key = PyRegion_staged_ref_ERR; + PyRegion_staged_ref_t staged_value = PyRegion_staged_ref_ERR; + staged_key = _PyRegion_STAGEREF(mp, key); + staged_value = _PyRegion_STAGEREF(mp, value); + if (staged_key == PyRegion_staged_ref_ERR || staged_value == PyRegion_staged_ref_ERR) { + goto Fail; + } + if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } else { + /* insertdict() handles any resizing that might be necessary */ + res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } - /* insertdict() handles any resizing that might be necessary */ - return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + if (res != 0) { + goto Fail; + } + + _PyRegion_CommitStagedRef(staged_key); + _PyRegion_CommitStagedRef(staged_value); + return 0; + +Fail: + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); + return -1; } int @@ -2762,8 +2881,10 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, STORE_VALUE(ep, NULL); STORE_HASH(ep, 0); } + _PyRegion_RemoveRef(_PyObject_CAST(mp), old_key); Py_DECREF(old_key); } + _PyRegion_RemoveRef(_PyObject_CAST(mp), old_value); Py_DECREF(old_value); ASSERT_CONSISTENT(mp); @@ -2912,11 +3033,13 @@ clear_lock_held(PyObject *op) if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, _PyObject_CAST(mp), oldkeys, IS_DICT_SHARED(mp)); } else { n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { + // This should never fail + _PyRegion_RemoveRef(op, oldvalues->values[i]); Py_CLEAR(oldvalues->values[i]); } if (oldvalues->embedded) { @@ -2926,7 +3049,7 @@ clear_lock_held(PyObject *op) set_values(mp, NULL); set_keys(mp, Py_EMPTY_KEYS); free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); + dictkeys_decref(interp, _PyObject_CAST(mp), oldkeys, false); } } ASSERT_CONSISTENT(mp); @@ -3089,6 +3212,7 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, *result = old_value; } else { + _PyRegion_REMOVEREF(mp, old_value); Py_DECREF(old_value); } return 1; @@ -3157,6 +3281,7 @@ PyDict_PopString(PyObject *op, const char *key, PyObject **result) } int res = PyDict_Pop(op, key_obj, result); + _PyRegion_REMOVEREF(op, key_obj); Py_DECREF(key_obj); return res; } @@ -3199,14 +3324,30 @@ dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, return NULL; } + PyRegion_staged_ref_t staged_key = PyRegion_staged_ref_ERR; + PyRegion_staged_ref_t staged_value = PyRegion_staged_ref_ERR; while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(mp); - return NULL; + // Check if the new references can be created + staged_key = _PyRegion_STAGEREF(mp, key); + staged_value = _PyRegion_STAGEREF(mp, value); + if (staged_key == PyRegion_staged_ref_ERR || staged_value == PyRegion_staged_ref_ERR) { + goto Fail; } + + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { + goto Fail; + } + + _PyRegion_CommitStagedRef(staged_key); + _PyRegion_CommitStagedRef(staged_value); } return mp; + +Fail: + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); + Py_DECREF(mp); + return NULL; } static PyDictObject * @@ -3225,6 +3366,7 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, } _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(iterable); + // FIXME(regions): xFrednet: Write Barrier is missing because FML while (_PySet_NextEntryRef(iterable, &pos, &key, &hash)) { if (insertdict(interp, mp, key, hash, Py_NewRef(value))) { Py_DECREF(mp); @@ -3270,6 +3412,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) it = PyObject_GetIter(iterable); if (it == NULL){ + // FIXME(regions): xFrednet: Does this need a WB? Py_DECREF(d); return NULL; } @@ -3278,6 +3421,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_BEGIN_CRITICAL_SECTION(d); while ((key = PyIter_Next(it)) != NULL) { status = setitem_lock_held((PyDictObject *)d, key, value); + _PyRegion_REMOVEREF(d, key); Py_DECREF(key); if (status < 0) { assert(PyErr_Occurred()); @@ -3289,6 +3433,7 @@ dict_iter_exit:; } else { while ((key = PyIter_Next(it)) != NULL) { status = PyObject_SetItem(d, key, value); + _PyRegion_REMOVEREF(d, key); Py_DECREF(key); if (status < 0) goto Fail; @@ -3297,10 +3442,12 @@ dict_iter_exit:; if (PyErr_Occurred()) goto Fail; + _PyRegion_REMOVELOCALREF(it); Py_DECREF(it); return d; Fail: + _PyRegion_REMOVELOCALREF(it); Py_DECREF(it); Py_DECREF(d); return NULL; @@ -3327,15 +3474,16 @@ dict_dealloc(PyObject *self) if (values != NULL) { if (values->embedded == 0) { for (i = 0, n = values->capacity; i < n; i++) { + _PyRegion_REMOVEREF(self, values->values[i]); Py_XDECREF(values->values[i]); } free_values(values, false); } - dictkeys_decref(interp, keys, false); + dictkeys_decref(interp, _PyObject_CAST(mp), keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys, false); + dictkeys_decref(interp, _PyObject_CAST(mp), keys, false); } if (Py_IS_TYPE(mp, &PyDict_Type)) { _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); @@ -3382,6 +3530,15 @@ dict_repr_lock_held(PyObject *self) // Prevent repr from deleting key or value during key format. Py_INCREF(key); Py_INCREF(value); + if (_PyRegion_AddLocalRef(key)) { + // Clear `value` to prevent a the `_PyRegion_RemoveLocalRef` call + // during error handling. + Py_CLEAR(value); + goto error; + } + if (_PyRegion_AddLocalRef(value)) { + goto error; + } if (!first) { // Write ", " @@ -3412,7 +3569,17 @@ dict_repr_lock_held(PyObject *self) goto error; } + if (_PyRegion_RemoveLocalRef(key)) { + // Clear the key to prevent a second remove ref call during + // error handling + Py_CLEAR(key); + goto error; + } Py_CLEAR(key); + if (_PyRegion_RemoveLocalRef(value)) { + Py_CLEAR(value); + goto error; + } Py_CLEAR(value); } @@ -3427,6 +3594,8 @@ dict_repr_lock_held(PyObject *self) error: Py_ReprLeave((PyObject *)mp); PyUnicodeWriter_Discard(writer); + _PyRegion_RemoveLocalRef(key); + _PyRegion_RemoveLocalRef(value); Py_XDECREF(key); Py_XDECREF(value); return NULL; @@ -3468,6 +3637,7 @@ dict_subscript(PyObject *self, PyObject *key) if (!PyDict_CheckExact(mp)) { /* Look up __missing__ method if we're a subclass. */ PyObject *missing, *res; + // FIXME(region): xFrednet: Write barrier for `missing` missing = _PyObject_LookupSpecial( (PyObject *)mp, &_Py_ID(__missing__)); if (missing != NULL) { @@ -3530,6 +3700,10 @@ keys_lock_held(PyObject *dict) PyObject *key; while (_PyDict_Next((PyObject*)mp, &pos, &key, NULL, NULL)) { assert(j < n); + if (_PyRegion_ADDLOCALREF(key)) { + Py_DECREF(v); + return NULL; + } PyList_SET_ITEM(v, j, Py_NewRef(key)); j++; } @@ -3579,6 +3753,10 @@ values_lock_held(PyObject *dict) PyObject *value; while (_PyDict_Next((PyObject*)mp, &pos, NULL, &value, NULL)) { assert(j < n); + if (_PyRegion_ADDLOCALREF(value)) { + Py_DECREF(v); + return NULL; + } PyList_SET_ITEM(v, j, Py_NewRef(value)); j++; } @@ -3616,6 +3794,9 @@ items_lock_held(PyObject *dict) */ again: n = mp->ma_used; + // Pyrona: We know that the list is new and therefore in the local region. + // This allows us to skip some write barriers and only requires LRC increases + // when we populate this array. v = PyList_New(n); if (v == NULL) return NULL; @@ -3641,6 +3822,10 @@ items_lock_held(PyObject *dict) while (_PyDict_Next((PyObject*)mp, &pos, &key, &value, NULL)) { assert(j < n); PyObject *item = PyList_GET_ITEM(v, j); + if (_PyRegion_ADDLOCALREFS(key, value)) { + Py_DECREF(v); + return NULL; + } PyTuple_SET_ITEM(item, 0, Py_NewRef(key)); PyTuple_SET_ITEM(item, 1, Py_NewRef(value)); j++; @@ -3732,6 +3917,10 @@ dict_update(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } +// ************************************************************************ +// Pyrona Write barrier barrier, above should be done +// ************************************************************************ + /* Update unconditionally replaces existing items. Merge has a 3rd argument 'override'; if set, it acts like Update, otherwise it leaves existing items unchanged. @@ -3869,7 +4058,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe return -1; ensure_shared_on_resize(mp); - dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, _PyObject_CAST(mp), mp->ma_keys, IS_DICT_SHARED(mp)); set_keys(mp, keys); STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); @@ -3902,6 +4091,18 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { int err = 0; + + // Check if the new references can be created + PyRegion_staged_ref_t staged_key = PyRegion_staged_ref_ERR; + PyRegion_staged_ref_t staged_value = PyRegion_staged_ref_ERR; + staged_key = _PyRegion_STAGEREF(mp, key); + staged_value = _PyRegion_STAGEREF(mp, value); + if (staged_key == PyRegion_staged_ref_ERR || staged_value == PyRegion_staged_ref_ERR) { + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); + return -1; + } + Py_INCREF(key); Py_INCREF(value); if (override == 1) { @@ -3917,6 +4118,8 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe else if (err > 0) { if (override != 0) { _PyErr_SetKeyError(key); + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); Py_DECREF(value); Py_DECREF(key); return -1; @@ -3926,8 +4129,11 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe } Py_DECREF(value); Py_DECREF(key); - if (err != 0) + if (err != 0) { + _PyRegion_ResetStagedRef(staged_key); + _PyRegion_ResetStagedRef(staged_value); return -1; + } if (orig_size != other->ma_keys->dk_nentries) { PyErr_SetString(PyExc_RuntimeError, @@ -6870,10 +7076,15 @@ _PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) else { dict = (PyDictObject *)PyDict_New(); } + + // TODO(Pyrona): Shouldn't this need error handling? if (_Py_IsImmutable(obj)) { // TODO(Immutable): For subinterpreters this will probably also need a lock! _PyImmutability_Freeze(_PyObject_CAST(dict)); + } else { + _PyRegion_ADDREF(obj, dict); } + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, dict); return dict; @@ -7266,11 +7477,12 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) } static void -clear_inline_values(PyDictValues *values) +clear_inline_values(PyObject *dict, PyDictValues *values) { if (values->valid) { FT_ATOMIC_STORE_UINT8(values->valid, 0); for (Py_ssize_t i = 0; i < values->capacity; i++) { + _PyRegion_RemoveRef(dict, values->values[i]); Py_CLEAR(values->values[i]); } } @@ -7286,7 +7498,7 @@ set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) Py_XINCREF(new_dict); FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); - clear_inline_values(values); + clear_inline_values(obj, values); } #ifdef Py_GIL_DISABLED @@ -7471,7 +7683,7 @@ detach_dict_from_object(PyDictObject *mp, PyObject *obj) } mp->ma_values = values; - invalidate_and_clear_inline_values(_PyObject_InlineValues(obj)); + invalidate_and_clear_inline_values(_PyObject_CAST(mp), _PyObject_InlineValues(obj)); assert(_PyObject_InlineValuesConsistencyCheck(obj)); ASSERT_CONSISTENT(mp); @@ -7490,7 +7702,7 @@ PyObject_ClearManagedDict(PyObject *obj) // We have no materialized dictionary and inline values // that just need to be cleared. // No dict to clear, we're done - clear_inline_values(_PyObject_InlineValues(obj)); + clear_inline_values(obj, _PyObject_InlineValues(obj)); return; } else if (FT_ATOMIC_LOAD_PTR_RELAXED(dict->ma_values) == @@ -7514,9 +7726,9 @@ PyObject_ClearManagedDict(PyObject *obj) PyDictKeysObject *oldkeys = dict->ma_keys; set_keys(dict, Py_EMPTY_KEYS); dict->ma_values = NULL; - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict)); + dictkeys_decref(interp, _PyObject_CAST(dict), oldkeys, IS_DICT_SHARED(dict)); STORE_USED(dict, 0); - clear_inline_values(_PyObject_InlineValues(obj)); + clear_inline_values(_PyObject_CAST(dict), _PyObject_InlineValues(obj)); Py_END_CRITICAL_SECTION(); } } @@ -7588,10 +7800,16 @@ ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) else { dict = PyDict_New(); } + + // FIXME(Pyrona): xFrednet: These should always succeed, but could fail + // some assumption failed. Maybe add error handling? if (_Py_IsImmutable(obj)) { // TODO(Immutable): For subinterpreters this will probably also need a lock! - _PyImmutability_Freeze(dict); + _PyImmutability_Freeze(_PyObject_CAST(dict)); + } else { + _PyRegion_ADDREF(obj, dict); } + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict); #ifdef Py_GIL_DISABLED done: @@ -7644,7 +7862,7 @@ void _PyDictKeys_DecRef(PyDictKeysObject *keys) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys, false); + dictkeys_decref(interp, NULL, keys, false); } static inline uint32_t diff --git a/Objects/object.c b/Objects/object.c index 75f154b51bbef2..6d8ddd5e03c648 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -26,6 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_region.h" // _PyRegion_SignalDealloc #include "pycore_symtable.h" // PySTEntry_Type #include "pycore_template.h" // _PyTemplate_Type _PyTemplateIter_Type #include "pycore_tuple.h" // _PyTuple_DebugMallocStats() @@ -1456,6 +1457,7 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternMortal(interp, &name); if (tp->tp_setattro != NULL) { + // TODO(regions): xFrednet: If type not regions aware => dirty if(Py_CHECKWRITE(v)){ err = (*tp->tp_setattro)(v, name, value); }else{ @@ -3127,6 +3129,7 @@ _PyTrash_thread_destroy_chain(PyThreadState *tstate) * up distorting allocation statistics. */ _PyObject_ASSERT(op, Py_REFCNT(op) == 0); + _PyRegion_SignalDealloc(op); (*dealloc)(op); } } @@ -3200,48 +3203,12 @@ _Py_Dealloc(PyObject *op) _PyTrash_thread_deposit_object(tstate, (PyObject *)op); return; } -#ifdef Py_DEBUG -#if !defined(Py_GIL_DISABLED) && !defined(Py_STACKREF_DEBUG) - /* This assertion doesn't hold for the free-threading build, as - * PyStackRef_CLOSE_SPECIALIZED is not implemented */ - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); -#endif - PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; - // Keep the old exception type alive to prevent undefined behavior - // on (tstate->curexc_type != old_exc_type) below - Py_XINCREF(old_exc); - // Make sure that type->tp_name remains valid - Py_INCREF(type); -#endif -#ifdef Py_TRACE_REFS - _Py_ForgetReference(op); -#endif _PyReftracerTrack(op, PyRefTracer_DESTROY); + _PyRegion_SignalDealloc(op); (*dealloc)(op); -#ifdef Py_DEBUG - // gh-89373: The tp_dealloc function must leave the current exception - // unchanged. - if (tstate != NULL && tstate->current_exception != old_exc) { - const char *err; - if (old_exc == NULL) { - err = "Deallocator of type '%s' raised an exception"; - } - else if (tstate->current_exception == NULL) { - err = "Deallocator of type '%s' cleared the current exception"; - } - else { - // It can happen if dealloc() normalized the current exception. - // A deallocator function must not change the current exception, - // not even normalize it. - err = "Deallocator of type '%s' overrode the current exception"; - } - _Py_FatalErrorFormat(__func__, err, type->tp_name); - } - Py_XDECREF(old_exc); - Py_DECREF(type); -#endif + if (tstate->delete_later && margin >= 4) { _PyTrash_thread_destroy_chain(tstate); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 70c55493a24b38..b10253c8a8c4f8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -17,6 +17,7 @@ #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_region.h" // _PyRegion_SignalDealloc() #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_typeobject.h" // struct type_cache #include "pycore_unicodeobject.h" // _PyUnicode_Copy @@ -2750,6 +2751,7 @@ subtype_dealloc(PyObject *self) /* Call the base tp_dealloc() */ assert(basedealloc); + _PyRegion_SignalDealloc(self); basedealloc(self); /* Can't reference self beyond this point. It's possible tp_del switched @@ -2859,6 +2861,7 @@ subtype_dealloc(PyObject *self) && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE)); assert(basedealloc); + _PyRegion_SignalDealloc(self); basedealloc(self); /* Can't reference self beyond this point. It's possible tp_del switched diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5c2308a012142a..1c56327556bdae 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -61,6 +61,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject_generated.h" // _PyUnicode_InitStaticStrings() +#include "immutability.h" // _PyImmutability_Freeze #include "stringlib/eq.h" // unicode_eq() #include // ptrdiff_t diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a1537dfde36266..cdfcb02da0391d 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -243,6 +243,7 @@ + @@ -264,6 +265,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index cdc27d33d5b234..7a40e11da3f6cf 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -328,6 +328,9 @@ Python + + Source Files + Source Files @@ -409,6 +412,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 661aeedc344036..3ac3c8135998e4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -285,6 +285,7 @@ + @@ -300,6 +301,7 @@ + @@ -375,6 +377,7 @@ + @@ -481,6 +484,7 @@ + @@ -640,6 +644,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 94bf7c0ff6cc7f..ef632a8598e524 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -694,6 +694,8 @@ Include\internal + Include\internal + Include\internal @@ -1054,6 +1056,8 @@ Modules + Modules + Modules @@ -1081,6 +1085,9 @@ Modules + + Modules + Modules @@ -1427,6 +1434,8 @@ Python + Python + Python @@ -1478,6 +1487,9 @@ Python + + Python + Python @@ -1523,6 +1535,9 @@ Python + + Python + Python diff --git a/Python/ceval.c b/Python/ceval.c index 7e76b53b94be2d..711d3d2d440cf9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -29,10 +29,12 @@ #include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_optimizer.h" // _PyUOpExecutor_Type +#include "pycore_ownership.h" // _PyOwnership_check_invariant #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject +#include "pycore_region.h" // _PyRegion_SignalDealloc #include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() @@ -90,6 +92,7 @@ if (_Py_DecRef_Immutable(op)) { \ _PyReftracerTrack(op, PyRefTracer_DESTROY); \ destructor dealloc = Py_TYPE(op)->tp_dealloc; \ + _PyRegion_SignalDealloc(op); \ (*dealloc)(op); \ } \ break; \ @@ -98,6 +101,7 @@ if ((--op->ob_refcnt) == 0) { \ _PyReftracerTrack(op, PyRefTracer_DESTROY); \ destructor dealloc = Py_TYPE(op)->tp_dealloc; \ + _PyRegion_SignalDealloc(op); \ (*dealloc)(op); \ } \ } while (0) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6d2383ac7c1c65..dede52f209a739 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -7,7 +7,7 @@ #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pystats.h" // _Py_PrintSpecializationStats() #include "pycore_runtime.h" // _PyRuntime - +#include "pycore_ownership.h" // _PyOwnership_check_invariant /* Notes about the implementation: @@ -1367,6 +1367,13 @@ _Py_HandlePending(PyThreadState *tstate) _PyThreadState_Attach(tstate); } +#ifdef Py_OWNERSHIP_INVARIANT + /* Check the region invariant if required. */ + if (_PyOwnership_check_invariant(tstate) != 0) { + return -1; + } +#endif + /* Pending signals */ if ((breaker & _PY_SIGNALS_PENDING_BIT) != 0) { if (handle_signals(tstate) != 0) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b3f3972344affa..d03a24f19b5586 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -150,13 +150,26 @@ do { \ /* Do interpreter dispatch accounting for tracing and instrumentation */ -#define DISPATCH() \ +#ifdef Py_OWNERSHIP_INVARIANT +# define DISPATCH() \ + { \ + if (_PyOwnership_check_invariant(tstate) != 0) { \ + JUMP_TO_LABEL(error); \ + } \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } +#else +# define DISPATCH() \ { \ assert(frame->stackpointer == NULL); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } +#endif #define DISPATCH_SAME_OPARG() \ { \ diff --git a/Python/errors.c b/Python/errors.c index e4e4d824df8d4c..d3476f928dce1a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -2070,8 +2070,8 @@ _PyErr_WriteToImmutable(PyObject* obj) #ifdef Py_DEBUG // Check if object has _freeeze_location attribute - if (PyObject_HasAttrString(obj, "__freeze_location__")) { - PyObject* freeze_location = PyObject_GetAttrString(obj, "__freeze_location__"); + if (PyObject_HasAttrString(obj, "__ownership_location__")) { + PyObject* freeze_location = PyObject_GetAttrString(obj, "__ownership_location__"); if (freeze_location != NULL) { // Load traceback module to convert to a format string diff --git a/Python/immutability.c b/Python/immutability.c index d52d33a268b1fe..84f690d181981e 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -5,10 +5,14 @@ #include #include "pycore_descrobject.h" #include "pycore_gc.h" -#include "pycore_object.h" #include "pycore_immutability.h" +#include "pycore_object.h" +#include "pycore_ownership.h" #include "pycore_list.h" +#include "pycore_region.h" +// Macro that jumps to error, if the expression `x` does not succeed. +#define SUCCEEDS(x) { do { int r = (x); if (r != 0) goto error; } while (0); } static PyObject * _destroy(PyObject* set, PyObject *objweakref) @@ -43,51 +47,14 @@ type_weakref(struct _Py_immutability_state *state, PyObject *obj) static int init_state(struct _Py_immutability_state *state) { - PyObject* frozen_importlib = NULL; - - frozen_importlib = PyImport_ImportModule("_frozen_importlib"); - if(frozen_importlib == NULL){ - return -1; - } - - state->module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); - if(state->module_locks == NULL){ - Py_DECREF(frozen_importlib); - return -1; - } - - state->blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); - if(state->blocking_on == NULL){ - Py_DECREF(frozen_importlib); - return -1; - } - state->freezable_types = PySet_New(NULL); if(state->freezable_types == NULL){ - Py_DECREF(frozen_importlib); return -1; } - Py_DECREF(frozen_importlib); - return 0; } -// This is separate to the previous init as it depends on the traceback -// module being available, and can cause a circular import if it is -// called during register freezable. -static -void init_traceback_state(struct _Py_immutability_state *state) -{ -#ifdef Py_DEBUG - PyObject *traceback_module = PyImport_ImportModule("traceback"); - if (traceback_module != NULL) { - state->traceback_func = PyObject_GetAttrString(traceback_module, "format_stack"); - Py_DECREF(traceback_module); - } -#endif -} - static struct _Py_immutability_state* get_immutable_state(void) { PyInterpreterState* interp = PyInterpreterState_Get(); @@ -152,10 +119,6 @@ static PyObject* pop(PyObject* s){ return item; } -static bool is_c_wrapper(PyObject* obj){ - return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); -} - // Lifted from Python/gc.c //******************************** */ #ifndef Py_GIL_DISABLED @@ -268,8 +231,7 @@ struct FreezeState { PyGC_Head visited_untracked; // Set of objects that have been visited and are immortal #endif PyObject* visited_list; // Some objects don't have GC space, so we need to track them separately. - - PyObject* dfs; // The DFS stack used to traverse the object graph during freezing. + struct _Py_immutability_state *imm_state; }; @@ -279,34 +241,39 @@ struct FreezeState { int init_freeze_state(struct FreezeState *state) { + state->imm_state = get_immutable_state(); + if (state->imm_state == NULL) { + return -1; + } + #ifndef Py_GIL_DISABLED gc_list_init(&(state->visited)); gc_list_init(&(state->visited_untracked)); #endif state->visited_list = NULL; - state->dfs = NULL; - - state->dfs = PyList_New(0); - if (state->dfs == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Failed to create DFS stack for freeze operation"); - return -1; - } - return 0; } -static inline void _Py_SetImmutable(PyObject *op) +static inline int _Py_SetImmutable(PyObject *op) { -if(op) { + if(op) { + SUCCEEDS(_PyRegion_SignalImmutable(op)); + #if SIZEOF_VOID_P > 4 op->ob_flags |= _Py_IMMUTABLE_FLAG; #else op->ob_refcnt |= _Py_IMMUTABLE_FLAG; #endif + } + + return 0; + +error: + return 1; } -int has_visited(struct FreezeState *state, PyObject *op) +int has_visited(PyObject *op, struct FreezeState *state) { // Not currently using state, but will need this for NoGIL builds. (void)state; @@ -326,7 +293,7 @@ add_visited_set(struct FreezeState *state, PyObject *op) #ifndef Py_GIL_DISABLED if (_PyObject_IS_GC(op)) { - _Py_SetImmutable(op); + SUCCEEDS(_Py_SetImmutable(op)); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_move(_Py_AS_GC(op), &(state->visited)); // Just set to space 0 for now. @@ -356,7 +323,7 @@ add_visited_set(struct FreezeState *state, PyObject *op) goto error; } - _Py_SetImmutable(op); + SUCCEEDS(_Py_SetImmutable(op)); return 0; error: @@ -368,8 +335,6 @@ add_visited_set(struct FreezeState *state, PyObject *op) // This unsets the immutability of all the objects that were visited. void fail_freeze(struct FreezeState *state) { - Py_XDECREF(state->dfs); - #ifndef Py_GIL_DISABLED PyGC_Head *gc; for (gc = _PyGCHead_NEXT(&(state->visited)); gc != &(state->visited); gc = _PyGCHead_NEXT(gc)) { @@ -430,191 +395,6 @@ void finish_freeze(struct FreezeState *state) #endif Py_XDECREF(state->visited_list); - Py_XDECREF(state->dfs); -} - -/** - * Special function for replacing globals and builtins with a copy of just what they use. - * - * This is necessary because the function object has a pointer to the global - * dictionary, and this is problematic because freezing any function directly - * (as we do with other objects) would make all globals immutable. - * - * Instead, we walk the function and find any places where it references - * global variables or builtins, and then freeze just those objects. The globals - * and builtins dictionaries for the function are then replaced with - * copies containing just those globals and builtins we were able to determine - * the function uses. - */ -static int shadow_function_globals(PyObject* op) -{ - PyObject* builtins = NULL; - PyObject* shadow_builtins = NULL; - PyObject* globals = NULL; - PyObject* shadow_globals = NULL; - PyFunctionObject* f = NULL; - PyObject* f_ptr = NULL; - PyCodeObject* f_code = NULL; - Py_ssize_t size; - bool check_globals = false; - - _PyObject_ASSERT(op, PyFunction_Check(op)); - - f = (PyFunctionObject*)op; - - globals = f->func_globals; - builtins = f->func_builtins; - - f_ptr = f->func_code; - - shadow_builtins = PyDict_New(); - if(shadow_builtins == NULL){ - goto nomemory; - } - - shadow_globals = PyDict_New(); - if(shadow_globals == NULL){ - goto nomemory; - } - - if(PyDict_SetItemString(shadow_globals, "__builtins__", shadow_builtins)){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - - _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); - f_code = (PyCodeObject*)f_ptr; - - size = 0; - if (f_code->co_names != NULL) - size = PySequence_Fast_GET_SIZE(f_code->co_names); - for(Py_ssize_t i = 0; i < size; i++){ - PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); - - if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ - // if the code calls the globals() builtin, then any - // cellvar or const in the function could, potentially, refer to - // a global variable. As such, we need to check if the globals - // dictionary contains that key and then make it immutable - // from this point forwards. - check_globals = true; - } - - if(PyDict_Contains(globals, name)){ - PyObject* value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(shadow_globals, name, value)){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - }else if(PyDict_Contains(builtins, name)){ - PyObject* value = PyDict_GetItem(builtins, name); - if(PyDict_SetItem(shadow_builtins, name, value)){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - } - } - - size = PySequence_Fast_GET_SIZE(f_code->co_consts); - for(Py_ssize_t i = 0; i < size; i++){ - PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); - if(check_globals && PyUnicode_Check(value)){ - // if the code calls the globals() builtin, then any - // cellvar or const in the function could, potentially, refer to - // a global variable. As such, we need to check if the globals - // dictionary contains that key and then make it immutable - // from this point forwards. - PyObject* name = value; - if(PyDict_Contains(globals, name)){ - value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(shadow_globals, name, value)){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - } - } - } - - size = 0; - if(f->func_closure != NULL){ - size = PyTuple_Size(f->func_closure); - if(size == -1){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - } - - for(Py_ssize_t i=0; i < size; ++i){ - PyObject* cellvar = PyTuple_GET_ITEM(f->func_closure, i); - PyObject* value = PyCell_GET(cellvar); - - PyObject* shadow_cellvar = PyCell_New(value); - if(PyTuple_SetItem(f->func_closure, i, shadow_cellvar) == -1){ - Py_DECREF(shadow_cellvar); - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - - if(PyUnicode_Check(value) && check_globals){ - // if the code calls the globals() builtin, then any - // cellvar or const in the function could, potentially, refer to - // a global variable. As such, we need to check if the globals - // dictionary contains that key and then make it immutable - // from this point forwards. - PyObject* name = value; - if(PyDict_Contains(globals, name)){ - value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(shadow_globals, name, value)){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return 0; - } - } - } - } - - f->func_globals = shadow_globals; - Py_DECREF(globals); - - f->func_builtins = shadow_builtins; - Py_DECREF(builtins); - - if(f->func_annotations == NULL){ - f->func_annotations = PyDict_New(); - if(f->func_annotations == NULL){ - goto nomemory; - } - } - - return 0; - -nomemory: - Py_XDECREF(shadow_builtins); - Py_XDECREF(shadow_globals); - PyErr_NoMemory(); - return -1; -} - -static int freeze_visit(PyObject* obj, void* dfs) -{ - if (obj == NULL) - return 0; - - if (_Py_IsImmutable(obj)) - return 0; - - if(push(dfs, obj)){ - PyErr_NoMemory(); - return -1; - } - - return 0; } static bool @@ -773,50 +553,43 @@ int _Py_DecRef_Immutable(PyObject *op) // Macro that jumps to error, if the expression `x` does not succeed. #define SUCCEEDS(x) { do { int r = (x); if (r != 0) goto error; } while (0); } -int traverse_freeze(PyObject* obj, PyObject* dfs) -{ - if(is_c_wrapper(obj)) { - // C functions are not mutable - // Types are manually traversed - return 0; - } +static +int freeze_check_obj(PyObject *obj, void *state_void) { + struct FreezeState *state = (struct FreezeState*)state_void; - // Function require some work to freeze, so we do not freeze the - // world as they mention globals and builtins. This will shadow what they - // use, and then we can freeze the those components. - if(PyFunction_Check(obj)){ - SUCCEEDS(shadow_function_globals(obj)); + // Immuable objects should be skipped + if (_Py_IsImmutable(obj)) { + return Py_OWNERSHIP_TRAVERSE_SKIP; } - if(PyType_Check(obj)){ - // TODO(Immutable): mjp: Special case for types not sure if required. We should review. - PyTypeObject* type = (PyTypeObject*)obj; - - SUCCEEDS(freeze_visit(type->tp_dict, dfs)); - SUCCEEDS(freeze_visit(type->tp_mro, dfs)); - // We need to freeze the tuple object, even though the types - // within will have been frozen already. - SUCCEEDS(freeze_visit(type->tp_bases, dfs)); - } - else - { - traverseproc traverse = Py_TYPE(obj)->tp_traverse; - if(traverse != NULL){ - SUCCEEDS(traverse(obj, (visitproc)freeze_visit, dfs)); - } + // Check if the object was already visited + if (has_visited(obj, state)) { + return Py_OWNERSHIP_TRAVERSE_SKIP; } - // The default tp_traverse will not visit the type object if it is - // not heap allocated, so we need to do that manually here to freeze - // the statically allocated types that are reachable. - if (!(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - SUCCEEDS(freeze_visit(_PyObject_CAST(Py_TYPE(obj)), dfs)); - } + // Check if the object can be frozen + SUCCEEDS(check_freezable(state->imm_state, obj)); - return 0; + // Mark the object as immutable and visited + SUCCEEDS(add_visited_set(state, obj)); + + // The object should be traversed, if everything passed until here + return Py_OWNERSHIP_TRAVERSE_VISIT; error: - return -1; + return Py_OWNERSHIP_TRAVERSE_ERR; +} + +static +int freeze_visit(PyObject *src, PyObject *obj, void *state_void) { + // The source is not needed in this function. This prevents warnings. + (void)src; + + if (_Py_IsImmutable(obj)) { + return Py_OWNERSHIP_TRAVERSE_SKIP; + } + + return Py_OWNERSHIP_TRAVERSE_VISIT; } // Main entry point to freeze an object and everything it can reach. @@ -825,80 +598,26 @@ int _PyImmutability_Freeze(PyObject* obj) if(_Py_IsImmutable(obj)){ return 0; } - int result = 0; - struct FreezeState freeze_state; // Initialize the freeze state + struct FreezeState freeze_state; SUCCEEDS(init_freeze_state(&freeze_state)); - struct _Py_immutability_state* state = get_immutable_state(); - if(state == NULL){ - goto error; - } - - -#ifdef Py_DEBUG - PyObject* freeze_location = NULL; - // In debug mode, we can set a freeze location for debugging purposes. - // Get a traceback object to use as the freeze location. - if (state->traceback_func == NULL) { - init_traceback_state(state); - } - - if (state->traceback_func != NULL) { - PyObject *stack = PyObject_CallFunctionObjArgs(state->traceback_func, NULL); - if (stack != NULL) { - // Add the type name to the top of the stack, can be useful. - PyObject* typename = PyObject_GetAttrString(_PyObject_CAST(Py_TYPE(obj)), "__name__"); - push(stack, typename); - freeze_location = stack; - } - } -#endif - - SUCCEEDS(push(freeze_state.dfs, obj)); - - while(PyList_Size(freeze_state.dfs) != 0){ - PyObject* item = pop(freeze_state.dfs); - - if(has_visited(&freeze_state, item)){ - continue; - } - - if(item == state->blocking_on || - item == state->module_locks){ - continue; - } - - SUCCEEDS(check_freezable(state, item)); - + // Traverse the object graph + SUCCEEDS(_PyOwnership_traverse_object_graph( + obj, #ifdef Py_DEBUG - if (freeze_location != NULL) { - // Some objects don't have attributes that can be set. - // As this is a Debug only feature, we could potentially increase the object - // size to allow this to be stored directly on the object. - if (PyObject_SetAttrString(item, "__freeze_location__", freeze_location) < 0) { - // Ignore failure to set _freeze_location - PyErr_Clear(); - // We still want to freeze the object, so we continue - } - } + false, /* freeze_location for debugging */ #endif - SUCCEEDS(add_visited_set(&freeze_state, item)); - - SUCCEEDS(traverse_freeze(item, freeze_state.dfs)); - } + freeze_check_obj, + freeze_visit, + (void*)&freeze_state + )); finish_freeze(&freeze_state); - goto finally; + return 0; error: fail_freeze(&freeze_state); - result = -1; - -finally: -#ifdef Py_DEBUG - Py_XDECREF(freeze_location); -#endif - return result; -} \ No newline at end of file + return -1; +} diff --git a/Python/ownership.c b/Python/ownership.c new file mode 100644 index 00000000000000..fb720a1e9e4527 --- /dev/null +++ b/Python/ownership.c @@ -0,0 +1,1033 @@ +#include "Python.h" +#include +#include "object.h" // _Py_IsImmutable +#include "pycore_descrobject.h" // _PyMethodWrapper_Type +#include "pycore_gc.h" // _PyGCHead_NEXT, _PyGCHead_PREV, _Py_FROM_GC +#include "pycore_interp.h" // PyThreadState_Get +#include "pycore_list.h" +#include "pycore_object.h" +#include "pycore_ownership.h" +#include "pycore_pyerrors.h" +#include "pycore_runtime.h" // _Py_ID +#include "pycore_region.h" // _PyRegion_Get(), Py_Region +#include "pycore_unicodeobject.h" +#include "pyerrors.h" +#include "refcount.h" + +// Macro that jumps to error, if the expression `x` does not succeed. +#define SUCCEEDS(x) { do { int r = (x); if (r != 0) goto error; } while (0); } + +#define _Py_region_data_CAST(region) _Py_CAST(_Py_region_data*, region) + +#define REGIO_SENTINEL_VALUE 0x12345678 + +static int init_state(_Py_ownership_state *state) +{ + state->module_locks = NULL; + state->blocking_on = NULL; + + state->tick = 2; +#ifdef Py_OWNERSHIP_INVARIANT + state->invariant_state = Py_OWNERSHIP_INVARIANT_DISABLED; +#endif + + return 0; +} + +static int init_import_state(_Py_ownership_state *state) { + PyObject* frozen_importlib = PyImport_ImportModule("_frozen_importlib"); + if (frozen_importlib == NULL) { + return -1; + } + + state->module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + if (state->module_locks == NULL) { + Py_DECREF(frozen_importlib); + return -1; + } + + state->blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + if (state->blocking_on == NULL) { + Py_DECREF(frozen_importlib); + return -1; + } + + Py_DECREF(frozen_importlib); + + // In debug mode, we can store the traceback for debugging purposes. + // Get a traceback object to use as the ownership location. +#ifdef Py_DEBUG + PyObject *traceback_module = PyImport_ImportModule("traceback"); + if (traceback_module != NULL) { + state->traceback_func = PyObject_GetAttrString(traceback_module, "format_stack"); + Py_DECREF(traceback_module); + } + + state->location_key = PyUnicode_FromString("__ownership_location__"); + if (state->location_key == NULL) { + return -1; + } + + PyInterpreterState *interp = PyInterpreterState_Get(); + if (interp == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get the interpreter state"); + return -1; + } + + _PyUnicode_InternImmortal(interp, &state->location_key); +#endif + + return 0; +} + +static _Py_ownership_state* get_ownership_state(void) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + if (interp == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get the interpreter state"); + return NULL; + } + + _Py_ownership_state *state = &interp->ownership; + if (state->tick == 0) { + if (init_state(state) == -1) { + PyErr_SetString(PyExc_RuntimeError, "Failed to initialize ownership state"); + return NULL; + } + } + + return state; +} + +static _Py_ownership_state* get_ownership_state_for_traverse(void) +{ + _Py_ownership_state* state = get_ownership_state(); + if (state == NULL) { + return NULL; + } + + if (state->blocking_on == NULL) { + if (init_import_state(state) != 0) { + PyErr_SetString(PyExc_RuntimeError, "Failed to initialize ownership state for traverse"); + return NULL; + } + } + + return state; +} + +#define IS_OPEN_REGION_TICK(tick) ((tick) % 2 == 0) + +Py_ssize_t _PyOwnership_get_current_tick(void) { + _Py_ownership_state* state = get_ownership_state(); + if (state == NULL) { + return 0; + } + + return state->tick; +} + +Py_ssize_t _PyOwnership_get_open_region_tick(void) { + _Py_ownership_state* state = get_ownership_state(); + if (state == NULL) { + return 0; + } + + // Only incremeant the counter, if the state is untrusted + if (!IS_OPEN_REGION_TICK(state->tick)) { + state->tick += 1; + + // Prevent overflow, by resetting early + if (state->tick > (PY_SSIZE_T_MAX - 10)) { + state->tick = 2; + } + } + assert(IS_OPEN_REGION_TICK(state->tick)); + + return state->tick; +} + +int _PyOwnership_notify_untrusted_code(void) { + _Py_ownership_state* state = get_ownership_state(); + if (state == NULL) { + return 1; + } + + // Only incremeant the counter, if the state is trusted + if (IS_OPEN_REGION_TICK(state->tick)) { + state->tick += 1; + } + assert(!IS_OPEN_REGION_TICK(state->tick)); + + // Everything is alright + return 0; +} + +/* This function returns true for C wrappers around functions, types and + * all kinds of wrappers around C with immutable state. For ownership these + * can be seen as immutable, meaning they can be referenced from immutable + * objects and from inside regions. + */ +int _PyOwnership_is_c_wrapper(PyObject* obj){ + return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); +} + +static int push(PyObject* s, PyObject* item) { + if (item == NULL) { + return 0; + } + + if (!PyList_Check(s)) { + PyErr_SetString(PyExc_TypeError, "Expected a list"); + return -1; + } + + return _PyList_AppendTakeRef(_PyList_CAST(s), Py_NewRef(item)); +} + +static PyObject* pop(PyObject* s) { + PyObject* item; + Py_ssize_t size = PyList_Size(s); + if (size == 0) { + return NULL; + } + + item = PyList_GetItem(s, size - 1); + if (item == NULL) { + return NULL; + } + + if (PyList_SetSlice(s, size - 1, size, NULL)) { + return NULL; + } + + return item; +} + +/** + * Special function for replacing globals and builtins with a copy of just what they use. + * + * This is necessary because the function object has a pointer to the global + * dictionary, and this is problematic because freezing any function directly + * (as we do with other objects) would make all globals immutable. + * + * Instead, we walk the function and find any places where it references + * global variables or builtins, and then freeze just those objects. The globals + * and builtins dictionaries for the function are then replaced with + * copies containing just those globals and builtins we were able to determine + * the function uses. + */ +static int shadow_function_globals(PyObject* op) +{ + PyObject* builtins = NULL; + PyObject* shadow_builtins = NULL; + PyObject* globals = NULL; + PyObject* shadow_globals = NULL; + PyFunctionObject* f = NULL; + PyObject* f_ptr = NULL; + PyCodeObject* f_code = NULL; + Py_ssize_t size; + bool check_globals = false; + + _PyObject_ASSERT(op, PyFunction_Check(op)); + + f = (PyFunctionObject*)op; + + globals = f->func_globals; + builtins = f->func_builtins; + + f_ptr = f->func_code; + + shadow_builtins = PyDict_New(); + if(shadow_builtins == NULL){ + goto nomemory; + } + + shadow_globals = PyDict_New(); + if(shadow_globals == NULL){ + goto nomemory; + } + + if(PyDict_SetItemString(shadow_globals, "__builtins__", shadow_builtins)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + + _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); + f_code = (PyCodeObject*)f_ptr; + + size = 0; + if (f_code->co_names != NULL) + size = PySequence_Fast_GET_SIZE(f_code->co_names); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); + + if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + check_globals = true; + } + + if(PyDict_Contains(globals, name)){ + PyObject* value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + }else if(PyDict_Contains(builtins, name)){ + PyObject* value = PyDict_GetItem(builtins, name); + if(PyDict_SetItem(shadow_builtins, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + } + } + + size = PySequence_Fast_GET_SIZE(f_code->co_consts); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); + if(check_globals && PyUnicode_Check(value)){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + PyObject* name = value; + if(PyDict_Contains(globals, name)){ + value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + } + } + } + + size = 0; + if(f->func_closure != NULL){ + size = PyTuple_Size(f->func_closure); + if(size == -1){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + } + + for(Py_ssize_t i=0; i < size; ++i){ + PyObject* cellvar = PyTuple_GET_ITEM(f->func_closure, i); + PyObject* value = PyCell_GET(cellvar); + + PyObject* shadow_cellvar = PyCell_New(value); + if(PyTuple_SetItem(f->func_closure, i, shadow_cellvar) == -1){ + Py_DECREF(shadow_cellvar); + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + + if(PyUnicode_Check(value) && check_globals){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + PyObject* name = value; + if(PyDict_Contains(globals, name)){ + value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return 0; + } + } + } + } + + f->func_globals = shadow_globals; + Py_DECREF(globals); + + f->func_builtins = shadow_builtins; + Py_DECREF(builtins); + + if(f->func_annotations == NULL){ + f->func_annotations = PyDict_New(); + if(f->func_annotations == NULL){ + goto nomemory; + } + } + + return 0; + +nomemory: + Py_XDECREF(shadow_builtins); + Py_XDECREF(shadow_globals); + PyErr_NoMemory(); + return -1; +} + +typedef struct ownership_traverse_state { + PyObject *source; + PyObject *dfs_stack; + + ownershipvisitproc caller_visit; + void *caller_state; +} ownership_traverse_state; + +static int ownership_visit(PyObject* target, void* traverse_state_void) +{ + // References to NULL can be ignored + if (target == NULL) + return 0; + + // Cast the state for easier access + ownership_traverse_state *traverse_state = + (ownership_traverse_state*)traverse_state_void; + + // Call the visit function + int result = (traverse_state->caller_visit)( + traverse_state->source, + target, + traverse_state->caller_state + ); + + // Enqueue the target if it should be traversed + if (result == Py_OWNERSHIP_TRAVERSE_VISIT) { + result = Py_OWNERSHIP_TRAVERSE_SKIP; + + if (push(traverse_state->dfs_stack, target)) { + PyErr_NoMemory(); + return -1; + } + } + + return result; +} + +/* This function calls the `visit` function for the fields of the `obj` + * which should be effected by ownership. The `data` pointer will be + * passed along as the second argument to `visit`. + */ +int _PyOwnership_traverse_obj(PyObject *obj, visitproc visit, void *data) { + if (PyType_Check(obj)) { + // TODO(Immutable): mjp: Special case for types not sure if required. We should review. + // Additional Note: xFrednet: It looks like each type has a handle to + // its module, so this is probably needed unless we want to freeze or + // replace the module pointer? + PyTypeObject* type = (PyTypeObject*)obj; + + SUCCEEDS(visit(type->tp_dict, data)); + SUCCEEDS(visit(type->tp_mro, data)); + // We need to freeze the tuple object, even though the types + // within will have been frozen already. + SUCCEEDS(visit(type->tp_bases, data)); + } + else + { + traverseproc traverse = Py_TYPE(obj)->tp_traverse; + if(traverse != NULL){ + SUCCEEDS(traverse(obj, visit, data)); + } + } + + // tp_traverse doesn't cover the object type, this therefore needs + // to explicitly visit the type. + SUCCEEDS(visit(_PyObject_CAST(Py_TYPE(obj)), data)); + + return 0; +error: + return -1; +} + +/* This prepares the given object to be frozen or moved into a region. The + * object is then traversed using `_PyOwnership_traverse_obj`. + */ +int _PyOwnership_prep_and_traverse_obj(PyObject* obj, void *data) +{ + if (_PyOwnership_is_c_wrapper(obj)) { + // C functions are not mutable + // Types are manually traversed + return 0; + } + + // Function require some work to freeze, so we do not freeze the + // world as they mention globals and builtins. This will shadow what they + // use, and then we can freeze the those components. + if (PyFunction_Check(obj)) { + SUCCEEDS(shadow_function_globals(obj)); + } + + SUCCEEDS(_PyOwnership_traverse_obj(obj, ownership_visit, data)); + + return 0; +error: + return -1; +} + +static int init_traverse_state( + ownership_traverse_state *state, + ownershipvisitproc caller_visit, + void *caller_state +) { + state->dfs_stack = NULL; + state->dfs_stack = PyList_New(0); + if (state->dfs_stack == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Failed to create DFS stack for object graph traversal"); + return -1; + } + + state->caller_visit = caller_visit; + state->caller_state = caller_state; + + return 0; +} + +/* This function traverses the object graph reachable from the given object. + * + * For every object it will call the `caller_check` function to determine if + * the object should be traversed. For every outgoing reference it will then + * call `caller_visit` which indicates if the referenced object should be + * traversed. + * + * This function will also store the current stacktrace in debug builds. + */ +int _PyOwnership_traverse_object_graph( + PyObject *obj, +#ifdef Py_DEBUG + int is_region_traversal, +#endif + ownershipcheckproc caller_check, + ownershipvisitproc caller_visit, + void *caller_state +) { + int result = 0; + +#ifdef Py_DEBUG + // This has to be declared early to support the `Py_XDECREF` if any of the + // `SUCCEEDS` fails + PyObject* location = NULL; +#endif + + // Enable the invariant. It has to be enabled at the beginning to allow + // reentry and failure in internal calls. + SUCCEEDS(_PyOwnership_invariant_enable()); + // This function incrementally marks new objects as frozen. During this + // process it is possible that frozen objects point to mutable ones. This + // therefore needs to pause the invariant. Otherwise we might get an + // exception when freezing calls into Python and triggers the invariant. + SUCCEEDS(_PyOwnership_invariant_pause()); + + // Initialize the traverse state + ownership_traverse_state traverse_state; + SUCCEEDS(init_traverse_state(&traverse_state, caller_visit, caller_state)); + + // Initialize ownership state + _Py_ownership_state *ownership_state = get_ownership_state_for_traverse(); + if (ownership_state == NULL) { + goto error; + } + +#ifdef Py_DEBUG + if (ownership_state->traceback_func != NULL) { + PyObject *stack = PyObject_CallFunctionObjArgs(ownership_state->traceback_func, NULL); + if (stack != NULL) { + // Add the type name to the top of the stack, can be useful. + PyObject* typename = PyObject_GetAttrString(_PyObject_CAST(Py_TYPE(obj)), "__name__"); + push(stack, typename); + location = stack; + + // Freezing the location allows all objects to reference it. + if (is_region_traversal) { + SUCCEEDS(_PyImmutability_Freeze(location)); + SUCCEEDS(_PyImmutability_Freeze(ownership_state->location_key)); + } + } + } +#endif + + // Push the current object to the pending stack + SUCCEEDS(push(traverse_state.dfs_stack, obj)); + + // While there is an object in the pending stack, check it + while(PyList_Size(traverse_state.dfs_stack) != 0){ + PyObject* item = pop(traverse_state.dfs_stack); + + // The `blocking_on` and `mutable_locks` should never be visited + if (item == ownership_state->blocking_on || + item == ownership_state->module_locks + ) { + continue; + } + +#ifdef Py_DEBUG + // Set the location early for freezing calles + if (location != NULL && !is_region_traversal) { + // Some objects don't have attributes that can be set. + // As this is a Debug only feature, we could potentially increase the object + // size to allow this to be stored directly on the object. + if (PyObject_SetAttr(item, ownership_state->location_key, location) < 0) { + // Ignore failure to set _freeze_location + PyErr_Clear(); + } + } +#endif + + switch (caller_check(item, caller_state)) { + // The object is fine, but shouldn't be traversed + case Py_OWNERSHIP_TRAVERSE_SKIP: + continue; + + // The object is okat and should be traversed + case Py_OWNERSHIP_TRAVERSE_VISIT: + traverse_state.source = item; + SUCCEEDS(_PyOwnership_prep_and_traverse_obj( + item, + (void*)&traverse_state)); + break; + + // An error occured + default: + goto error; + } + +#ifdef Py_DEBUG + // Set the location late for region calles + if (location != NULL && is_region_traversal) { + if (PyObject_SetAttr(item, ownership_state->location_key, location) < 0) { + // Ignore failure to set _freeze_location + PyErr_Clear(); + } + } +#endif + } + + goto finally; + +error: + result = -1; + +finally: +#ifdef Py_DEBUG + Py_XDECREF(location); +#endif + Py_XDECREF(traverse_state.dfs_stack); + // Indicate that this funciton no longer requires the invariant to be paused. + // This can't use the `SUCCEEDS` macro, since that one would jump to the + // `error` label above. + if (_PyOwnership_invariant_resume() != 0) { + result = -1; + } + return result; +} + +// All code belonging to the invariant +#ifdef Py_OWNERSHIP_INVARIANT + +static void throw_invariant_error( + PyObject* src, + PyObject* tgt, + const char *format_str, + PyObject *format_arg +) { + // Don't stomp existing exception + PyThreadState *tstate = PyThreadState_Get(); + if (!tstate || _PyErr_Occurred(tstate)) { + return; + } + + // Create the error, this sets the error value in `tstate` + PyErr_Format(PyExc_RuntimeError, format_str, format_arg); + + // Set source and target fields + // Get the current exception (should be a RuntimeError) + PyObject *exc = PyErr_GetRaisedException(); + assert(exc && PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_RuntimeError)); + + // printf("Source %p in %x (Is immutable: %d)\n", src, _PyRegion_Get(src), _Py_IsImmutable(src)); + // printf("Source Type %p: %s\n", src->ob_type, src->ob_type->tp_name); + // printf("Target %p in %x\n", tgt, _PyRegion_Get(tgt)); + // printf("Target Type %p: %s\n", tgt->ob_type, tgt->ob_type->tp_name); + // Add 'source' and 'target' attributes to the exception + PyObject_SetAttr(exc, &_Py_ID(source), src ? src : Py_None); + PyObject_SetAttr(exc, &_Py_ID(target), tgt ? tgt : Py_None); + + PyErr_SetRaisedException((PyObject*)exc); +} + +// Lifted from Python/gc.c +//******************************** */ +typedef struct _gc_runtime_state GCState; +#define GEN_HEAD(gcstate, n) ((n == 0) ? (&(gcstate)->young.head) : (&(gcstate)->old[n - 1].head)) +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV +#define FROM_GC _Py_FROM_GC +//******************************** */ + +typedef struct _check_invariant_state { + PyObject *src; + // A list of regions which have been checked during this pass. + Py_region_t regions; +} _check_invariant_state; + +static void _check_invariant_state_track(_check_invariant_state *state, Py_region_t region) { + if (region == _Py_LOCAL_REGION + || region == _Py_IMMUTABLE_REGION + || region == _Py_COWN_REGION + ) { + return; + } + + _Py_region_data *data = _Py_region_data_CAST(region); + + // Each region should only be added once + if (data->invariant_data.next != NULL_REGION) { + return; + } + + // Add the region to the linked list + data->invariant_data.next = state->regions; + state->regions = region; +} + +static int validate_check_invariant_state(_check_invariant_state* state) { + // Validate the visited region + Py_region_t region = state->regions; + while (region != REGIO_SENTINEL_VALUE) { + _Py_region_data *data = _Py_region_data_CAST(region); + + if (_PyRegion_IsDirty(region)) { + // Dirty regions can be checked, if PY_OWNERSHIP_INVARIANT_CHECK_DIRTY is set + const char* env = Py_GETENV("PY_OWNERSHIP_INVARIANT_CHECK_DIRTY"); + if (!env) { + goto next; + } + } + + if ((data->invariant_data.lrc != 0 || data->invariant_data.osc != 0) + && !_PyRegion_IsOpen(region) + ) { + throw_invariant_error( + data->bridge, NULL, + "Invariant Error: References into `source` were found, but the region is closed", + Py_None); + return -1; + } + + // This value is just an upper bound, since there can be references + // from non GC objects, for example on the stack + if (data->lrc < data->invariant_data.lrc) { + throw_invariant_error( + data->bridge, NULL, + "Invariant Error: The LRC of the region in `source` is too high", + Py_None); + return -1; + } + + // This value is just an upper bound, since there can be references + // from non GC objects, for example on the stack + if (data->osc < data->invariant_data.osc) { + throw_invariant_error( + data->bridge, NULL, + "Invariant Error: The OSC of the region in `source` is too high", + Py_None); + return -1; + } + + next: + // Get the next region + region = data->invariant_data.next; + } + + return 0; +} + +static void clear_check_invariant_state(_check_invariant_state* state) { + // Clear temporary region data + while (state->regions != REGIO_SENTINEL_VALUE) + { + _Py_region_data *data = _Py_region_data_CAST(state->regions); + + // Get the next region + state->regions = data->invariant_data.next; + + // Clear data + data->invariant_data.lrc = 0; + data->invariant_data.osc = 0; + data->invariant_data.next = NULL_REGION; + } +} + +static int check_invariant_validate_immutable(PyObject* obj) { + // Immutable objects should be in the immutable region + if (_PyRegion_Get(obj) != _Py_IMMUTABLE_REGION) { + throw_invariant_error( + obj, NULL, + "Invariant Error: Immutable objects should be in the immutable region", + Py_None); + return -1; + } + + return 0; +} + +static int check_invariant_visit_immutable(PyObject* tgt, _check_invariant_state* state) { + PyObject* src = state->src; + + // C wrappers are special and allowed + if (_PyOwnership_is_c_wrapper(tgt)) { + return 0; + } + + // Make sure the immutable source only points to immutable objects + if (!_Py_IsImmutable(tgt)) { + throw_invariant_error( + src, tgt, + "Invariant Error: An immutable objects points to a mutable one", + Py_None); + return -1; + } + + return 0; +} + +static int check_invariant_visit_owned(PyObject* tgt, _check_invariant_state* state) { + PyObject* src = state->src; + + Py_region_t src_region = _PyRegion_Get(src); + Py_region_t tgt_region = _PyRegion_Get(tgt); + + // This should never happen, since immutable objects have their own visit + // funciton + assert(src_region != _Py_IMMUTABLE_REGION); + + // C wrappers are special and allowed + if (_PyOwnership_is_c_wrapper(tgt)) { + return 0; + } + + // References to objects in the cown and immutable regions are allowed + if (tgt_region == _Py_IMMUTABLE_REGION || tgt_region == _Py_COWN_REGION) { + return 0; + } + + // Intra-region references are allowed + if (src_region == tgt_region) { + return 0; + } + + _check_invariant_state_track(state, tgt_region); + + // Dirty regions are basically allowed to do anything + if (_PyRegion_IsDirty(src_region)) { + // Dirty regions can be checked, if PY_OWNERSHIP_INVARIANT_CHECK_DIRTY is set + const char* env = Py_GETENV("PY_OWNERSHIP_INVARIANT_CHECK_DIRTY"); + if (!env) { + return 0; + } + } + + // Objects inside a region are not allowed to reference local objects + if (tgt_region == _Py_LOCAL_REGION) { + throw_invariant_error( + src, tgt, + "Invariant Error: A owned object is referencing a local object", Py_None); + return -1; + } + + // If the object references another region, it has to be the bridge object + // and this object needs to be the parent. + if (!_PyRegion_IsBridge(tgt)) { + throw_invariant_error( + src, tgt, + "Invariant Error: A owned object is referencing a foreign contained object", + Py_None); + return -1; + } + + // This is the owning reference to the target region, but target doesn't know about it + if (_PyRegion_IsBridge(tgt) && !_PyRegion_IsParent(tgt_region, src_region)) { + throw_invariant_error( + src, tgt, + "Invariant Error: A sub region doesn't know about it's parent", + Py_None); + return -1; + } + + // Update the invariant OSC to check the source region data + if (_PyRegion_IsBridge(tgt) && _PyRegion_IsOpen(tgt_region)) { + _Py_region_data *src_data = _Py_region_data_CAST(src_region); + src_data->invariant_data.osc += 1; + } + + return 0; +} +static int check_invariant_visit_local(PyObject* tgt, _check_invariant_state* state) { + PyObject* src = state->src; + + Py_region_t src_region = _PyRegion_Get(src); + Py_region_t tgt_region = _PyRegion_Get(tgt); + + // This should never happen, since immutable objects have their own visit + // funciton + assert(src_region == _Py_LOCAL_REGION); + + // References to static regions are trivially fine + if (tgt_region == _Py_LOCAL_REGION + || tgt_region == _Py_IMMUTABLE_REGION + || tgt_region == _Py_COWN_REGION + ) { + return 0; + } + + _check_invariant_state_track(state, tgt_region); + + _Py_region_data *tgt_data = _Py_region_data_CAST(tgt_region); + tgt_data->invariant_data.lrc += 1; + + return 0; +} + +int _PyOwnership_check_invariant(PyThreadState *tstate) { + _Py_ownership_state *ownership_state = get_ownership_state(); + if (ownership_state == NULL) { + return -1; + } + + // Only run the invariant if it's actully enabled and there is no + // function which paused the invariant + if (ownership_state->invariant_state != Py_OWNERSHIP_INVARIANT_ENABLED) { + return 0; + } + + // Don't run during shutdown. Python needs to mutate data in this state + // and any breakage will not really matter, since this universe is at + // its end. + if (Py_IsFinalizing()) { + ownership_state->invariant_state = Py_OWNERSHIP_INVARIANT_DISABLED; + return 0; + } + + // Don't stomp existing exceptions + if (_PyErr_Occurred(tstate)) { + return 0; + } + + int result = 0; + + // Use the GC data to find all the objects, and traverse them to + // confirm all their references satisfy the invariant. + GCState *gcstate = &tstate->interp->gc; + + _check_invariant_state check_state = { + .src = NULL, + .regions = REGIO_SENTINEL_VALUE + }; + + // There is an cyclic doubly linked list per generation of all the objects + // in that generation. + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + PyGC_Head *containers = GEN_HEAD(gcstate, i); + PyGC_Head *gc = GC_NEXT(containers); + // Walk doubly linked list of objects. + for (; gc != containers; gc = GC_NEXT(gc)) { + PyObject *ob = FROM_GC(gc); + + // C wrappers are complicated see description of the called + // function. We treat them as immutable objects. But we + // don't traverse them. + if (_PyOwnership_is_c_wrapper(ob)) { + continue; + } + + // Prepare the check state + check_state.src = ob; + + // Select which validation function should be used, based on the + // current object. + visitproc visit = NULL; + if (_Py_IsImmutable(ob)) { + check_invariant_validate_immutable(ob); + visit = (visitproc)check_invariant_visit_immutable; + } else if (!_Py_IsLocal(ob)) { + _check_invariant_state_track(&check_state, _PyRegion_Get(ob)); + visit = (visitproc)check_invariant_visit_owned; + } else if (_Py_IsLocal(ob)) { + visit = (visitproc)check_invariant_visit_local; + } + + // Use traverse proceduce to visit each field of the object. + SUCCEEDS(_PyOwnership_traverse_obj(ob, visit, &check_state)); + } + } + + SUCCEEDS(validate_check_invariant_state(&check_state)); + + goto finally; + +error: + // Disable the invariant + ownership_state->invariant_state = Py_OWNERSHIP_INVARIANT_DISABLED; + // Return -1 to indicate an error + result = -1; + +finally: + clear_check_invariant_state(&check_state); + return result; +} + +int _PyOwnership_invariant_enable(void) { + _Py_ownership_state *state = get_ownership_state(); + if (state == NULL) { + return -1; + } + + if (state->invariant_state == Py_OWNERSHIP_INVARIANT_DISABLED) { + state->invariant_state = Py_OWNERSHIP_INVARIANT_ENABLED; + } + + return 0; +} + +int _PyOwnership_invariant_disable(void) { + _Py_ownership_state *state = get_ownership_state(); + if (state == NULL) { + return -1; + } + + state->invariant_state = Py_OWNERSHIP_INVARIANT_DISABLED; + + return 0; +} + +int _PyOwnership_invariant_pause(void) { + _Py_ownership_state *state = get_ownership_state(); + if (state == NULL) { + return -1; + } + + if (state->invariant_state != Py_OWNERSHIP_INVARIANT_DISABLED) { + state->invariant_state += 1; + } + + return 0; +} + +int _PyOwnership_invariant_resume(void) { + _Py_ownership_state *state = get_ownership_state(); + if (state == NULL) { + return -1; + } + + if (state->invariant_state != Py_OWNERSHIP_INVARIANT_DISABLED) { + state->invariant_state -= 1; + } + + return 0; +} +#endif /* Py_OWNERSHIP_INVARIANT */ diff --git a/Python/pystate.c b/Python/pystate.c index 9bb6d92890c21e..168d3ce6f9430f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -791,11 +791,16 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(interp->imports.importlib == NULL); assert(interp->imports.import_func == NULL); - Py_CLEAR(interp->immutability.module_locks); - Py_CLEAR(interp->immutability.blocking_on); Py_CLEAR(interp->immutability.freezable_types); Py_CLEAR(interp->immutability.destroy_cb); + interp->ownership.tick = 0; + Py_CLEAR(interp->ownership.module_locks); + Py_CLEAR(interp->ownership.blocking_on); +#ifdef Py_OWNERSHIP_INVARIANT + interp->ownership.invariant_state = Py_OWNERSHIP_INVARIANT_DISABLED; +#endif + Py_CLEAR(interp->sysdict_copy); Py_CLEAR(interp->builtins_copy); Py_CLEAR(interp->dict); diff --git a/Python/region.c b/Python/region.c new file mode 100644 index 00000000000000..2fefa33f6c58dd --- /dev/null +++ b/Python/region.c @@ -0,0 +1,1854 @@ +#include "Python.h" +#include "refcount.h" +#include "pyerrors.h" + +#include "pycore_interp.h" // PyThreadState_Get +#include "pycore_ownership.h" +#include "pycore_pyerrors.h" +#include "pycore_region.h" +#include "pycore_runtime.h" // _Py_ID +#include "pycore_list.h" + +#include + +/* Macro that jumps to error, if the expression `x` does not succeed. */ +#define SUCCEEDS(x) { do { int r = (x); if (r != 0) goto error; } while (0); } + +/* Checks for predefined static regions without data */ +#define IS_LOCAL_REGION(r) ((Py_region_t)(r) == _Py_LOCAL_REGION) +#define IS_IMMUTABLE_REGION(r) ((Py_region_t)(r) == _Py_IMMUTABLE_REGION) +#define IS_COWN_REGION(r) ((Py_region_t)(r) == _Py_COWN_REGION) +#define HAS_DATA(r) (!IS_LOCAL_REGION(r) && !IS_IMMUTABLE_REGION(r) && !IS_COWN_REGION(r)) + +/* Magic values for `_Py_region_data.open_tick` */ +#define OPEN_TICK_CLOSED 0 +#define OPEM_TICK_DIRTY 1 + +/* Macros to access the owner and check for tags */ +#define OWNER_TAG_COWN ((Py_uintptr_t)0b01) +#define OWNER_TAG_MERGED ((Py_uintptr_t)0b10) +#define OWNER_TAG_MERGE_PENDING ((Py_uintptr_t)0b11) +#define OWNER_TAG_MASK (OWNER_TAG_COWN | OWNER_TAG_MERGED) +#define OWNER_PTR_MASK (~OWNER_TAG_MASK) +#define GET_OWNER_WITH_TAG(data) (((_Py_region_data*)(data))->owner) +#define GET_OWNER_PTR(data) (GET_OWNER_WITH_TAG(data) & OWNER_PTR_MASK) +#define HAS_OWNER_TAG(data, tag) ((GET_OWNER_WITH_TAG(data) & OWNER_TAG_MASK) == tag) + +/* Helper macros */ +#define ASSERT_IS_UNION_ROOT(region) assert(!HAS_DATA(region) || !HAS_OWNER_TAG(region, OWNER_TAG_MERGED)) +#define ASSERT_REGION_HAS_NO_TAG(region) assert((region & OWNER_PTR_MASK) == region) + +#define STAGED_REF_NOP_MERGE ((Py_uintptr_t)0xbeef) +#define STAGED_REF_LRC_TAG ((Py_uintptr_t)0x1) +#define STAGED_TAG_MASK (STAGED_REF_LRC_TAG) +#define STAGED_PTR_MASK (~STAGED_REF_LRC_TAG) +#define STAGED_HAS_TAG(staged, tag) ((staged & STAGED_TAG_MASK) == tag) +#define STAGED_AS_PTR(staged) (staged & STAGED_PTR_MASK) + +// Prototyes +static int regiondata_inc_osc(Py_region_t region); +static int regiondata_dec_osc(Py_region_t region); +static int regiondata_is_open(Py_region_t data); +static Py_region_t regiondata_get_parent(Py_region_t region); +static int regiondata_set_parent(Py_region_t region, Py_region_t new_parent); +static int regiondata_check_status(Py_region_t region); + +static PyObject* list_pop(PyObject* s){ + PyObject* item; + Py_ssize_t size = PyList_Size(s); + if(size == 0){ + return NULL; + } + item = PyList_GetItem(s, size - 1); + if(item == NULL){ + return NULL; + } + if(PyList_SetSlice(s, size - 1, size, NULL)){ + return NULL; + } + return item; +} + +// Lifted from Python/gc.c +//******************************** */ +#ifndef Py_GIL_DISABLED +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV + +static inline void +gc_set_old_space(PyGC_Head *g, int space) +{ + assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); + g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; + g->_gc_next |= space; +} + +static inline void +gc_list_init(PyGC_Head *list) +{ + // List header must not have flags. + // We can assign pointer by simple cast. + list->_gc_prev = (uintptr_t)list; + list->_gc_next = (uintptr_t)list; +} + +static inline int +gc_list_is_empty(PyGC_Head *list) +{ + return (list->_gc_next == (uintptr_t)list); +} + +/* Move `node` from the gc list it's currently in (which is not explicitly + * named here) to the end of `list`. This is semantically the same as + * gc_list_remove(node) followed by gc_list_append(node, list). + */ +static void +gc_list_move(PyGC_Head *node, PyGC_Head *list) +{ + /* Unlink from current list. */ + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + + /* Relink at end of new list. */ + // list must not have flags. So we can skip macros. + PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; + _PyGCHead_SET_PREV(node, to_prev); + _PyGCHead_SET_NEXT(to_prev, node); + list->_gc_prev = (uintptr_t)node; + _PyGCHead_SET_NEXT(node, list); +} + +/* append list `from` onto list `to`; `from` becomes an empty list */ +static void +gc_list_merge(PyGC_Head *from, PyGC_Head *to) +{ + assert(from != to); + if (!gc_list_is_empty(from)) { + PyGC_Head *to_tail = GC_PREV(to); + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + assert(from_head != from); + assert(from_tail != from); + + _PyGCHead_SET_NEXT(to_tail, from_head); + _PyGCHead_SET_PREV(from_head, to_tail); + + _PyGCHead_SET_NEXT(from_tail, to); + _PyGCHead_SET_PREV(to, from_tail); + } + gc_list_init(from); +} + +static struct _gc_runtime_state* +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} +#endif // Py_GIL_DISABLED +// ********************************************************************** + +// This uses the given arguments to create and throw a `RegionError` +static void throw_region_error( + const char *format_str, PyObject *format_args, + PyObject* src, PyObject* tgt) +{ + // Don't stomp existing exception + PyThreadState *tstate = PyThreadState_Get(); + if (_PyErr_Occurred(tstate)) { + return; + } + + PyErr_Format(PyExc_RuntimeError, format_str, format_args); + + // Set source and target fields + // Get the current exception (should be a RuntimeError) + PyObject *exc = PyErr_GetRaisedException(); + assert(exc && PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_RuntimeError)); + + // Add 'source' and 'target' attributes to the exception + PyObject_SetAttr(exc, &_Py_ID(source), src ? src : Py_None); + PyObject_SetAttr(exc, &_Py_ID(target), tgt ? tgt : Py_None); + + PyErr_SetRaisedException((PyObject*)exc); +} + +static Py_region_t regiondata_new(void) { + _Py_region_data* data = (_Py_region_data*)calloc(1, sizeof(_Py_region_data)); + if (data == NULL) { + return NULL_REGION; + } + + gc_list_init(&data->gc_list); + data->rc = 1; + return (Py_region_t)data; +} + +static void regiondata_inc_rc(Py_region_t region) { + if (!HAS_DATA(region)) { + return; + } + + // Change RC + _Py_region_data *data = (_Py_region_data*)region; + data->rc += 1; +} + +static void regiondata_dec_rc(Py_region_t region) { + if (!HAS_DATA(region)) { + return; + } + + // Change RC + _Py_region_data *data = (_Py_region_data*)region; + data->rc -= 1; + + // Dealloc if needed + if (data->rc == 0) { + // The RC should never hit zero with a cown as the parent + assert(HAS_OWNER_TAG(data, OWNER_TAG_COWN) == 0); + + // The region has to be closed, when the RC hits zero + assert(data->open_tick == OPEN_TICK_CLOSED); + + // Decrement the owner RC, the owner will always be a region at + // this point. This accesses the owner directly since we want + // to decrement the RC of this specific owning region not the + // root of the union find. + regiondata_dec_rc(GET_OWNER_PTR(region)); + + // Free the data belonging to this region + free(data); + } +} + +/* Returns the root of the union-find tree that the given region is a part of + */ +static Py_region_t regiondata_union_root(Py_region_t region, bool *update_region) { + // Regions without data are always roots of the union-find forest + if (!HAS_DATA(region)) { + return region; + } + + // Act like the merge worked out + if (HAS_OWNER_TAG(region, OWNER_TAG_MERGE_PENDING)) { + *update_region = false; + return regiondata_union_root(GET_OWNER_PTR(region), update_region); + } + + // Return if this if the root of the union-find + if (!HAS_OWNER_TAG(region, OWNER_TAG_MERGED)) { + return region; + } + + // Increase the RC of `region` to avoid special casing in the following code + regiondata_inc_rc(region); + + // Keep the child pointer to reassign the owner and correct the RC + _Py_region_data *child = (_Py_region_data*)region; + region = GET_OWNER_PTR(region); + + // Walk the union-find until the root is reached + while (HAS_DATA(region) && HAS_OWNER_TAG(region, OWNER_TAG_MERGED)) { + // Assign the owner of the child. This halves the tree everytime the + // root is search for. This results in an amortized time of O(1). + child->owner = GET_OWNER_WITH_TAG(region); + + // The RC of the `_Py_region_data` which was previously the owner of + // `child` has to be decremented. However, this might deallocate + // the object. This code therefore wait until the next iteration + // when the `region` is stored in `child` to decrement the RC. + regiondata_dec_rc((Py_region_t)child); + + // Prepare `child` and `region` values for the next iteration. + child = (_Py_region_data*)region; + region = GET_OWNER_PTR(region); + } + + // Cleanup RC count + regiondata_dec_rc((Py_region_t)child); + + // The `region` value now holds the root of the union-find tree. + return region; +} + +// FIXME: xFrednet: If performance of this becomes a problem, we could write a +// specialized version for merging into static regions as this makes several +// operations easier. The compiler could figure several of these out, but it +// would require several layers of inlining. +static int regiondata_union_merge( + Py_region_t source, Py_region_t target +) { + // Invariant: + assert(HAS_DATA(source)); + ASSERT_IS_UNION_ROOT(source); + ASSERT_IS_UNION_ROOT(target); + + // Clear the pending tag if present + _Py_region_data *source_data = (_Py_region_data*) source; + if (HAS_OWNER_TAG(source, OWNER_TAG_MERGE_PENDING)) { + Py_region_t pending_target = GET_OWNER_PTR(source); + assert(pending_target == target); + regiondata_dec_rc(pending_target); + source_data->owner = NULL_REGION; + } + ASSERT_REGION_HAS_NO_TAG(target); + + int result = 0; + + // Increase the RC of `target` to make sure none of the following + // operations deallocates it by accident. + regiondata_inc_rc(target); + + // If the target was open, we increment the OSC by one to keep it + // open until this merge is done. This makes sure that a region + // doesn't get closed and reopened. + bool cleanup_inc_osc = false; + if (regiondata_is_open(target)) { + // Inc OSC can't fail here, since `target` is already open + regiondata_inc_osc(target); + cleanup_inc_osc = true; + } + + // If `target` is the parent of `source` it can be merged. This unsets + // the parent of `source` to correctly update the OSC and RC. + Py_region_t source_parent = regiondata_get_parent(source); + if (source_parent == target) { + // Set parent can't fail here, since this function has increased the + // OSC, thereby keeping the region open if it was previously open. + regiondata_set_parent(source, NULL_REGION); + source_parent = NULL_REGION; + } + + // `source` can't be merged if it has any other parent than `target` + // as the link from `source_parent` to the bridge of `source` would + // break isolation after the merge. However, a merge of source into + // target is always allowed. + if (source_parent != NULL_REGION && !IS_IMMUTABLE_REGION(target)) { + // TODO: xFrednet: Better error message with explaination and conditional + // based on if X is static + throw_region_error( + "unable to merge X into Y since X still has a parent", Py_None, + Py_None, Py_None); + goto error; + } + + // Set the owner to the target with the merged tag + regiondata_inc_rc(target); + source_data->owner = target | OWNER_TAG_MERGED; + + // Update the bridge object + if (source_data->bridge) { + regiondata_dec_rc(source_data->bridge->region); + source_data->bridge->region = NULL_REGION; + } + + // Merge stats into the `target` + if (HAS_DATA(target)) { + _Py_region_data *target_data = (_Py_region_data*)target; + target_data->lrc += source_data->lrc; + target_data->osc += source_data->osc; + gc_list_merge(&source_data->gc_list, &target_data->gc_list); + + // Check how the `open_tick` should be updated + if (target_data->open_tick == OPEN_TICK_CLOSED) { + // The target was previously closed, merging the new data + // might have opened it. Taking the `open_tick` from `source` + // puts target into the right state. + target_data->open_tick = source_data->open_tick; + } else if (source_data->open_tick == OPEN_TICK_CLOSED) { + // It's fine if the target is open but source is closed + } else if (target_data->open_tick != source_data->open_tick) { + // At least one of the regions was dirty since the `open_tick` + // is mismatching. + target_data->open_tick = OPEM_TICK_DIRTY; + } else { + // The open ticks are equal, nothing needs to be done + } + + // Check if the region can be opened or closed. + regiondata_check_status(target); + } else if (IS_LOCAL_REGION(target)) { + struct _gc_runtime_state* gc_state = get_gc_state(); + // Use `old[0]` here, we are setting the visited space to 0 in add_visited_set(). + gc_list_merge(&(source_data->gc_list), &(gc_state->old[0].head)); + } + + // Remove information from `source` + source_data->bridge = NULL; + source_data->lrc = 0; + source_data->osc = 0; + source_data->open_tick = OPEN_TICK_CLOSED; + + assert(gc_list_is_empty(&source_data->gc_list)); + + // Skip the error label and run the normal cleanup code + goto cleanup; + +error: + result = 1; + +cleanup: + // This returns the OSC which was aquired ealier to keep it open during + // this merge. + if (cleanup_inc_osc) { + result |= regiondata_dec_osc(target); + } + + // Decrement the `target` RC again + regiondata_dec_rc(target); + + return result; +} + +/* This opens the region and marks it as clean. + * + * This operation may fail if: + * - The `_Py_ownership_state` is currently unavailable + * - Opening a parent region failed + * - TODO: xFrednet: If the owing cown is released. + */ +static int regiondata_open(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Regions without metadata are always open + if (!HAS_DATA(region)) { + return 0; + } + + // Don't reopen a open region, as that would mark it as clean again + if (regiondata_is_open(region)) { + return 0; + } + + // Mark the region as open. + _Py_region_data *data = (_Py_region_data*)region; + data->open_tick = _PyOwnership_get_open_region_tick(); + + // Check if opening the region was successful + if (data->open_tick == OPEN_TICK_CLOSED) { + return 1; + } + + // The open tick should always be even, see invariant + assert((data->open_tick % 2) == 0); + + // Notify the owner + if (HAS_OWNER_TAG(region, OWNER_TAG_COWN)) { + // TODO: xFrednet: Implement this branch, probably just an assert + assert(false); + } else if (regiondata_get_parent(region) != 0) { + SUCCEEDS(regiondata_open(regiondata_get_parent(region))); + } + + // Check for failure, which would leave the region closed + return 0; + +error: + // Mark the region as closed on failure. + data->open_tick = OPEN_TICK_CLOSED; + return 1; +} + +static int regiondata_mark_as_clean(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + assert(regiondata_is_open(region)); + + // Regions without metadata are always clean + if (!HAS_DATA(region)) { + return 0; + } + + // Mark the region as open. + _Py_region_data *data = (_Py_region_data*)region; + Py_ssize_t old_open_tick = data->open_tick; + data->open_tick = _PyOwnership_get_open_region_tick(); + + // Check if an error occured + if (data->open_tick == OPEN_TICK_CLOSED) { + data->open_tick = old_open_tick; + return -1; + } + + // The open tick should always be even, see invariant + assert((data->open_tick % 2) == 0); + + return 0; +} + +static int regiondata_is_open(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Regions without metadata are always open + if (!HAS_DATA(region)) { + return true; + } + + return ((_Py_region_data*)region)->open_tick != OPEN_TICK_CLOSED; +} + +static void regiondata_mark_as_dirty(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Regions without metadata are never dirty + if (!HAS_DATA(region)) { + return; + } + + // Only open regions can be marked as dirty + assert(regiondata_is_open(region)); + + // Mark region as dirty + _Py_region_data* data = (_Py_region_data*)region; + data->open_tick = OPEM_TICK_DIRTY; +} + +static int regiondata_is_dirty(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Regions without metadata are never dirty + if (!HAS_DATA(region)) { + return false; + } + + // Closed regions are always clean + if (!regiondata_is_open(region)) { + return false; + } + + // Check if the region is open and already marked as dirty + _Py_region_data* data = (_Py_region_data*)region; + if (data->open_tick == OPEM_TICK_DIRTY) { + return true; + } + + // Check if untrusted code was called since this region was opened + Py_ssize_t current_tick = _PyOwnership_get_current_tick(); + if (data->open_tick == current_tick) { + return false; + } + + // Set to dirty constant for quicker lookup + regiondata_mark_as_dirty(region); + + return true; +} + +/* This closes the region and propagates the status to the owner. + * + * This operation may fail if: + * - The region is dirty (potentially caused by `_Py_ownership_state` being unavailable) + * - Closing a parent region failed + * - TODO: xFrednet: If the owing cown is released. + * + * The region might still be closed, if the error came from an owner. + */ +static int regiondata_close(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + assert(regiondata_is_open(region)); + + // Regions without metadata can't be closed + if (!HAS_DATA(region)) { + return 0; + } + + // Dirty regions can't be closed + if (regiondata_is_dirty(region)) { + return 1; + } + + // Mark the region as closed. + _Py_region_data *data = (_Py_region_data*)region; + data->open_tick = OPEN_TICK_CLOSED; + + // Notify the owner + if (HAS_OWNER_TAG(region, OWNER_TAG_COWN)) { + // TODO: xFrednet: Implement this branch + assert(false); + } else if (regiondata_get_parent(region) != 0) { + return regiondata_close(regiondata_get_parent(region)); + } + + // Check for failure, which would leave the region closed + return 0; +} + +static int regiondata_closes_after_lrc(Py_region_t region, Py_ssize_t lrc) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions can't be closed + if (!HAS_DATA(region)) { + return 0; + } + + // Return 0 if the region will be kept open, even if the LRC is adjusted + _Py_region_data *data = (_Py_region_data*)region; + if (regiondata_is_dirty(region) && data->osc > 0) { + return 0; + } + + // Return true, if the known local references are the only ones keeping + // the region open + if (data->lrc == lrc) { + return 1; + } + + // Invariant, the LRC should never be less than the known LRC + assert(data->lrc >= lrc); + + return 0; +} + +/* This uses the inner state of the region and closes it if possible. + * + * This can fail if the region gets closed, see `regiondata_close`. + */ +static int regiondata_check_close(Py_region_t region) { + // Check if the region should be closed at this point. + if (regiondata_closes_after_lrc(region, 0)) { + return regiondata_close(region); + } + + // Nothing needs to be done, and everything is fine + return 0; +} + +/* This uses the inner state of the region to check if it needs to be opened. + * + * This can fail if the region gets opened, see `regiondata_open`. + */ +static int regiondata_check_open(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions can't be opened + if (!HAS_DATA(region)) { + return 0; + } + + // Check if the region can currently be closed + // - LRC and OSC can be negative if the region is staged (waiting to be merged) + _Py_region_data *data = (_Py_region_data*)region; + if (data->lrc > 0 || data->osc > 0) { + // Propagate the result + return regiondata_open(region); + } + + // Nothing needs to be done, and everything is fine + return 0; +} + +/* This uses the inner state of the region to check if it should be opened + * or closed + */ +static int regiondata_check_status(Py_region_t region) { + if (regiondata_is_open(region)) { + return regiondata_check_close(region); + } else { + return regiondata_check_open(region); + } +} + +/* This increases the local reference count. + * + * This might open this and parent regions, which can fail. See + * `regiondata_open` for possible failures. + * */ +static int regiondata_inc_lrc(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions don't need to be updated + if (!HAS_DATA(region)) { + return 0; + } + + // Attempt to mark the region as open + if (regiondata_open(region)) { + return 1; + } + + // Update the LRC, once the region is open + _Py_region_data *data = (_Py_region_data*)region; + data->lrc += 1; + + // This is a hack, by marking every region as dirty we force + // every region to be closed by cleaning it. + _PyRegion_HackDirtyForPrototype(region); + + return 0; +} + +/* This decreases the local reference count. + * + * This might close this and parent regions, which can fail. See + * `regiondata_close` for possible failures. + * */ +static int regiondata_dec_lrc(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions don't need to be updated + if (!HAS_DATA(region)) { + return 0; + } + + // Update the LRC + _Py_region_data *data = (_Py_region_data*)region; + if (data == 0) { + // Open the region, to mark it as dirty + SUCCEEDS(regiondata_open(region)); + regiondata_mark_as_dirty(region); + } else { + data->lrc -= 1; + + // Check the region state to determine if it should be closed. + SUCCEEDS(regiondata_check_close(region)); + } + + // Return 0 on success + return 0; + +error: + // Undo the LRC decrement + data->lrc += 1; + + // Propagate the failure information + return 1; +} + +/* This increases the open-subregion count. (This does not update RC) + * + * This might open this and parent regions, which can fail. See + * `regiondata_open` for possible failures. + * */ +static int regiondata_inc_osc(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions don't need to be updated + if (!HAS_DATA(region)) { + return 0; + } + + // Attempt to mark the region as open + if (regiondata_open(region)) { + return 1; + } + + // Update the OSC, once the region is open + _Py_region_data *data = (_Py_region_data*)region; + data->osc += 1; + + return 0; +} + +/* This decreases the open-subregion count. (This does not update RC) + * + * This might close this and parent regions, which can fail. See + * `regiondata_close` for possible failures. + * */ +static int regiondata_dec_osc(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions don't need to be updated + if (!HAS_DATA(region)) { + return 0; + } + + // Update the OSC + _Py_region_data *data = (_Py_region_data*)region; + data->osc -= 1; + + // Check the region state to determine if it should be closed. + SUCCEEDS(regiondata_check_close(region)); + + // Return 0 on success + return 0; + +error: + // Undo the OSC decrement + data->osc += 1; + + // Propagate the failure information + return 1; +} + +/* Setting the parent of an open region, might open the new parent region + * and close the old parent region. + * + * This can fail, see `regiondata_open` and `regiondata_close` for possible + * failures. + * */ +static int regiondata_set_parent(Py_region_t region, Py_region_t new_parent) { + // Check invariant: + assert(HAS_DATA(region)); + ASSERT_REGION_HAS_NO_TAG(new_parent); + ASSERT_IS_UNION_ROOT(region); + ASSERT_IS_UNION_ROOT(new_parent); + assert(region != new_parent); + ASSERT_REGION_HAS_NO_TAG(GET_OWNER_WITH_TAG(region)); + + // Get the old parent + _Py_region_data* data = (_Py_region_data*) region; + Py_region_t old_parent = GET_OWNER_PTR(data); + + // Notify the parents, if this region is open. + if (regiondata_is_open(region)) { + if (regiondata_inc_osc(new_parent)) { + return 1; + } + + if (regiondata_dec_osc(old_parent)) { + // Undo the inc_osc from above. + regiondata_dec_osc(new_parent); + return 1; + } + } + + // Only set the parent here, once all the failable operations are done + data->owner = new_parent; + regiondata_inc_rc(new_parent); + regiondata_dec_rc(old_parent); + + return 0; +} + +/* Returns the pointer to the parent region or 0 if the region doesn't have a + * parent. + */ +static Py_region_t regiondata_get_parent(Py_region_t region) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + + // Static regions never have a parent + if (!HAS_DATA(region)) { + return 0; + } + + // Don't return the owner, if it's a cown + if (HAS_OWNER_TAG(region, OWNER_TAG_COWN)) { + return 0; + } + + // Get the parent + bool update_region = true; + Py_region_t parent_field = GET_OWNER_PTR(region); + Py_region_t parent_root = regiondata_union_root(parent_field, &update_region); + + // If the parent was merged with another region we want to update the + // owner to point at the root. + if (parent_field != parent_root && update_region) { + _Py_region_data* data = (_Py_region_data*) region; + data->owner = parent_root; + regiondata_inc_rc(parent_root); + regiondata_dec_rc(parent_field); + } + + // Get the root of the parent + return parent_root; +} + +/* Returns `true` if the given region has a parent + */ +static bool regiondata_has_parent(Py_region_t region) { + return regiondata_get_parent(region) != 0; +} + +/* Returns true, if `other` is an ancestor of `region`. + */ +static bool regiondata_is_ancestor(Py_region_t region, Py_region_t ancestor) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + ASSERT_IS_UNION_ROOT(ancestor); + ASSERT_REGION_HAS_NO_TAG(ancestor); + + // Static regions never have parents + if (!HAS_DATA(region)) { + return false; + } + + // Static regions are never parents + if (!HAS_DATA(ancestor)) { + return false; + } + + // Walk the ancestor tree until the root + while (region) { + if (region == ancestor) { + return true; + } + + region = regiondata_get_parent(region); + } + + return false; +} + +/* Returns true if the given object is the bridge of the given region + */ +static bool regiondata_is_bridge(Py_region_t region, PyObject *obj) { + // Invariant: + ASSERT_IS_UNION_ROOT(region); + assert(obj != NULL); + + // Static regions have no brigde objects + if (!HAS_DATA(region)) { + return false; + } + + _Py_region_data *data = (_Py_region_data*)region; + + return _PyObject_CAST(data->bridge) == obj; +} + +/* Sets the region of the object to the newly given region. + * + * This will just update the RC of the old and new region, all other state, + * like the LRC, has to be updated separatly. + */ +static void _PyRegion_Set(PyObject* obj, Py_region_t new_region) { + // Invariant: + assert(obj); + ASSERT_IS_UNION_ROOT(new_region); + ASSERT_REGION_HAS_NO_TAG(new_region); + + // Remove the object from its GC list. This has to be done before the + // region update to make sure that the list head remains allocated + if (HAS_DATA(new_region) && PyObject_IS_GC(obj) && PyObject_GC_IsTracked(obj)) { + _Py_region_data *data = (_Py_region_data *)new_region; + gc_set_old_space(_Py_AS_GC(obj), 0); + gc_list_move(_Py_AS_GC(obj), &data->gc_list); + } + + // Update the region and region rc + Py_region_t old_region = obj->ob_region; + obj->ob_region = new_region; + regiondata_inc_rc(new_region); + regiondata_dec_rc(old_region); +} + +// Add the transitive closure of objects in the local region reachable from obj to region +// static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) {} +typedef struct AddRegionState { + Py_region_t merge_region; + Py_region_t subject_region; + PyObject *open_subregion_list; +} AddRegionState; + +static +int _add_to_region_check_obj(PyObject *obj, void *state_void) { + // Sanity Check, all objects given to this function should act like they're + // in the subject region + assert(_PyRegion_Get(obj) == ((AddRegionState*)state_void)->subject_region); + + // `_add_to_region_visit` already does the filtering and ensures that only + // new objects are traversed. This is therefore a no-op indicateing that + // the object should be traversed. + return Py_OWNERSHIP_TRAVERSE_VISIT; +} + +#include "immutability.h" + +static +int _add_to_region_visit(PyObject *src, PyObject *tgt, void *state_void) { + AddRegionState *state = (AddRegionState*)state_void; + + Py_region_t tgt_region = _PyRegion_Get(tgt); + + // These regerences are allowed and should not be followed + if (IS_IMMUTABLE_REGION(tgt_region) || IS_COWN_REGION(tgt_region)) { + return Py_OWNERSHIP_TRAVERSE_SKIP; + } + + _Py_region_data *merge_data = (_Py_region_data*)state->merge_region; + + // Immortal object have no real RC, this makes it infeasable to have them + // in a region and dynamically track their ownership. Immortal objects are + // intended to be immutable in Python, so it should be safe to implicitly + // freeze them. + if (_Py_IsImmortal(tgt)) { + assert(IS_LOCAL_REGION(tgt_region) && "At this point it would have to be local"); + + // Check if we can just freeze it + if (_PyImmutability_Freeze(tgt) != 0) { + // Clear the error from freezing and throw our own + PyErr_Clear(); + + throw_region_error( + "An immportal object can't be part of a region, and implicit freezing failed", + Py_None, src, tgt); + return Py_OWNERSHIP_TRAVERSE_ERR; + } + + return Py_OWNERSHIP_TRAVERSE_SKIP; + } + + // Take ownership of local objects + if (IS_LOCAL_REGION(tgt_region)) { + // Add incoming references to the LRC + // -1 for the reference this call came from + // + // FIXME(regions): xFrednet: Handle weak references + merge_data->lrc += Py_REFCNT(tgt) - 1; + + // Add the object to the merge region, this will also prevent it + // from being traversed again. + _PyRegion_Set(tgt, state->merge_region); + + // Return and notify that `tgt` should also be traversed + return Py_OWNERSHIP_TRAVERSE_VISIT; + } + + // The target was previously in the local region but has already been + // added to the merge region by a previous iteration. This therefore only + // adjusts the LRC + if (tgt_region == state->subject_region) { + // The LRC of the merge region can go negative by this operation as + // this also includes references which should be subtract from the + // LRC of the subject region. + merge_data->lrc -= 1; + // Problem, dictionary gets populated by the set attribute, the visit then + // subtracts for this reference. Just freeze dict and don't use the default + // write barrier for population. + + // The object should not be traversed. + return Py_OWNERSHIP_TRAVERSE_SKIP; + } + + // At this point, we know that target is in another region. + // If target is in a different region, it has to be a bridge object. + // References to contained objects are forbidden. + if (!regiondata_is_bridge(tgt_region, tgt)) { + // TODO: Better error message + throw_region_error("References to objects in other regions are forbidden", Py_None, src, tgt); + + return Py_OWNERSHIP_TRAVERSE_ERR; + } + + // The target is a bridge object from another region. This is allowed, if + // the region doesn't have a parent + if (regiondata_has_parent(tgt_region)) { + // TODO: Better error message + throw_region_error("Regions are not allowed to have multiple parents", Py_None, src, tgt); + + return Py_OWNERSHIP_TRAVERSE_ERR; + } + + // This region can become the parent of the target region, but this is + // not allowed to create a cycle + if (regiondata_is_ancestor(state->subject_region, tgt_region)) { + // TODO: Better error message + throw_region_error("Regions are not allowed to create cycles in the ancestor tree", Py_None, src, tgt); + + return Py_OWNERSHIP_TRAVERSE_ERR; + } + + // From the previous checks it is know that `tgt` is the bridge object + // of a free region. Thus we can make it a sub region and allow the + // reference. + // + // `regiondata_set_parent` will also ensure that the `osc` is updated. + regiondata_set_parent(tgt_region, state->merge_region); + if (state->open_subregion_list && regiondata_is_open(tgt_region)) { + if (_PyList_AppendTakeRef( + _PyList_CAST(state->open_subregion_list), Py_NewRef(tgt))) + { + return Py_OWNERSHIP_TRAVERSE_ERR; + } + } + + // The object reference was accepted, but the target should not be traversed + return Py_OWNERSHIP_TRAVERSE_SKIP; +} + +/* Attempts to add the given `targets` to the `subject_region`. The interal + * state is updated accordingly. + * + * The `src` argument is only used for error reporting and can be NULL. + * + * FIXME(regions): xFrednet: Optional, this could be specialized for cases + * which are known to succeed, to more the objects directly into the subject + * region. + */ +PyRegion_staged_ref_t regiondata_stage_objects( + Py_region_t subject_region, PyObject* src, + int tgt_count, PyObject **targets, + PyObject* open_subregion_list) +{ + // Invariant: + ASSERT_IS_UNION_ROOT(subject_region); + if (tgt_count == 0) { + return STAGED_REF_NOP_MERGE; + } + + // Enable and pause invariant + SUCCEEDS(_PyOwnership_invariant_enable()); + SUCCEEDS(_PyOwnership_invariant_pause()); + + int result = 0; + PyRegion_staged_ref_t staged_res = STAGED_REF_NOP_MERGE; + + // Initialize the state + AddRegionState add_state; + add_state.subject_region = subject_region; + add_state.merge_region = regiondata_new(); + add_state.open_subregion_list = open_subregion_list; + if (add_state.merge_region == NULL_REGION) { + PyErr_NoMemory(); + goto error; + } + _Py_region_data* merge_data = (_Py_region_data*)add_state.merge_region; + regiondata_inc_rc(subject_region); + merge_data->owner = (subject_region | OWNER_TAG_MERGE_PENDING); + + for (int tgt_i = 0; tgt_i < tgt_count; tgt_i += 1) { + PyObject *tgt = targets[tgt_i]; + + // Manually call visit with `tgt` as the target to ensure that it is + // correctly added to the merge region or throws an error + result = _add_to_region_visit(src, tgt, (void*)&add_state); + + switch (result) + { + case Py_OWNERSHIP_TRAVERSE_VISIT: + // Traverse the object graph + SUCCEEDS(_PyOwnership_traverse_object_graph( + tgt, +#ifdef Py_DEBUG + true, /* freeze_location for debugging */ +#endif + _add_to_region_check_obj, + _add_to_region_visit, + (void*)&add_state)); + case Py_OWNERSHIP_TRAVERSE_SKIP: + // Indicate success + result = 0; + break; + default: + goto error; + } + } + + // Return the staged region to be commited later + staged_res = (PyRegion_staged_ref_t)add_state.merge_region; + goto finally; + +error: + // Merge the region into local, to undo any ownership changes + regiondata_union_merge(add_state.merge_region, _Py_LOCAL_REGION); + staged_res = PyRegion_staged_ref_ERR; + // Ignoring the error, since an error will already be reported + _PyOwnership_invariant_resume(); + +finally: + return staged_res; +} + +void staged_ref_reset(PyRegion_staged_ref_t staged_ref) { + // Error reporting is done by the staging step. This can therefore + // just ignore the error. + if (staged_ref == PyRegion_staged_ref_ERR) { + return; + } + + // Everything is fine + if (staged_ref == STAGED_REF_NOP_MERGE) { + return; + } + + // The LRC has to be decremented + if (STAGED_HAS_TAG(staged_ref, STAGED_REF_LRC_TAG)) { + Py_region_t region = STAGED_AS_PTR(staged_ref); + int res = regiondata_dec_lrc(region); + assert(res == 0); + return; + } + + // Merge the pending region into local + Py_region_t staged_region = STAGED_AS_PTR(staged_ref); + assert(HAS_OWNER_TAG(staged_region, OWNER_TAG_MERGE_PENDING)); + + // This should never fail + int res = regiondata_union_merge(staged_region, _Py_LOCAL_REGION); + assert(res == 0); + regiondata_dec_rc(staged_region); + + res = _PyOwnership_invariant_resume(); + assert(res == 0); +} + +void staged_ref_commit(PyRegion_staged_ref_t staged_ref) { + assert(staged_ref != PyRegion_staged_ref_ERR); + + // Everything is fine + if (staged_ref == STAGED_REF_NOP_MERGE) { + return; + } + + // The LRC was already incremented and can stay that way + if (STAGED_HAS_TAG(staged_ref, STAGED_REF_LRC_TAG)) { + return; + } + + // Mark the region as merged + Py_region_t staged_region = STAGED_AS_PTR(staged_ref); + assert(HAS_OWNER_TAG(staged_region, OWNER_TAG_MERGE_PENDING)); + Py_region_t target = GET_OWNER_PTR(staged_region); + + // This should never fail + int res = regiondata_union_merge(staged_region, target); + assert(res == 0); + regiondata_dec_rc(staged_region); + + res = _PyOwnership_invariant_resume(); + assert(res == 0); +} + +/* Simple wrapper to call `regiondata_add_object` with one target */ +PyRegion_staged_ref_t regiondata_stage_object(Py_region_t subject_region, PyObject* src, PyObject *target) { + return regiondata_stage_objects(subject_region, src, 1, &target, NULL); +} + +/* Simple wrapper to call `regiondata_add_object` with one target */ +PyRegion_staged_ref_t regiondata_add_object(Py_region_t subject_region, PyObject* src, PyObject *target) { + // Stage the references to be addeds + PyRegion_staged_ref_t staged_ref = regiondata_stage_object(subject_region, src, target); + if (staged_ref == PyRegion_staged_ref_ERR) { + return -1; + } + + // Should always succeed + staged_ref_commit(staged_ref); + return 0; +} + +int regiondata_clean(PyObject* bridge) { + // Invariant + ASSERT_IS_UNION_ROOT(_PyRegion_Get(bridge)); + assert(HAS_DATA(_PyRegion_Get(bridge))); + + int result = 0; + PyObject *pending_list = NULL; + + // We only need to close a region which is open + if (!regiondata_is_open(_PyRegion_Get(bridge))) { + return 0; + } + + // Incrementing the RC of the bridge will ensure that we don't + // accidentally release a cown early + if (regiondata_inc_lrc(_PyRegion_Get(bridge))) { + return -1; + } + Py_INCREF(bridge); + + // Enable and pause invariant + SUCCEEDS(_PyOwnership_invariant_enable()); + SUCCEEDS(_PyOwnership_invariant_pause()); + + // Initialize the state + pending_list = PyList_New(1); + if (pending_list == NULL) { + goto error; + } + PyList_SET_ITEM(_PyList_CAST(pending_list), 0, Py_NewRef(bridge)); + + while(PyList_Size(pending_list) != 0){ + PyObject* item = list_pop(pending_list); + Py_region_t item_region = _PyRegion_Get(item); + + assert(regiondata_is_bridge(item_region, item)); + // TODO: Account in LRC for reference from owner, if present. + + // Store metadata for the new region + assert(HAS_DATA(item_region)); + Py_region_t owner = ((_Py_region_data*)item_region)->owner; + ((_Py_region_data*)item_region)->owner = 0; + bool was_open = regiondata_is_open(item_region); + + // Merge the region into local + if (regiondata_union_merge(item_region, _Py_LOCAL_REGION)) { + regiondata_mark_as_dirty(item_region); + Py_DECREF(item); + goto error; + } + + // Create the new clean region + Py_region_t clean_region = regiondata_new(); + if (clean_region == NULL_REGION) { + goto error; + } + + PyRegion_staged_ref_t staged_ref = regiondata_stage_objects( + clean_region, NULL, 1, &item, pending_list); + if (staged_ref == PyRegion_staged_ref_ERR) { + regiondata_dec_rc(clean_region); + goto error; + } + staged_ref_commit(staged_ref); + + // TODO(regions): xFrednet: This doesn't account for region union... + // + // `stage_objects` accounts for a reference from a contained object to + // the added object, mening that the LRC is missing a count of 1 here. + // We increment the LRC if it doesn't have a owner. + if (owner == 0) { + SUCCEEDS(regiondata_inc_lrc(clean_region)); + // FIXME(regions): xFrednet: The hack currently marks the region as + // dirty when the LRC is increased. the following function should + // no longer be used when all barriers are in place + SUCCEEDS(regiondata_mark_as_clean(clean_region)); + } + + // The region should now be marked as clean + assert(!regiondata_is_dirty(clean_region)); + + // Refill metadata. + _Py_region_data* clean_region_data = (_Py_region_data*)clean_region; + clean_region_data->owner = owner; + clean_region_data->bridge = _PyBridgeObject_CAST(item); + clean_region_data->bridge->region = clean_region; // Move RC ownership + if (!was_open && regiondata_is_open(clean_region)) { + regiondata_inc_osc(clean_region); + } + } + + goto finally; +error: + result = -1; + +finally: + // Decrease the LRC, which was incremented at the start to keep the region + // open. This shoudln't close the region, since the bridge object should + // only be borrowed. + regiondata_dec_lrc(_PyRegion_Get(bridge)); + Py_DECREF(bridge); + Py_XDECREF(pending_list); + // Resume invariant + _PyOwnership_invariant_resume(); + return result; +} + +/* ==================================== + * Exported functions + * ==================================== + */ + + +/* Returns the region of the given object. This is the slow path of `_PyRegion_`. + * + * This function can't be inlined as it requires additional metadata to check + * if the region of the object was merged with another one. + */ +Py_region_t _PyRegion_GetSlow(PyObject *obj) { + // Immutable objects can be shared across threads, it's not save to access + // the region information without synchronization. + if (_Py_IsImmutable(obj)) { + return _Py_IMMUTABLE_REGION; + } + + bool update_region = true; + Py_region_t region = regiondata_union_root(obj->ob_region, &update_region); + + // Check if the region should be updated, this can happen if the object + // region was merged into another region. + if (obj->ob_region != region && update_region) { + _PyRegion_Set(obj, region); + } + + return region; +} + +/* Creates a new region and moves the bridge object into it. The new region + * will be returned. + */ +int _PyRegion_New(_PyBridgeObject *bridge) { + Py_region_t region = regiondata_new(); + if (region == NULL_REGION) { + return -1; + } + + _Py_region_data *data = (_Py_region_data*)region; + + // A weak reference, the bridge will clear this pointer when it is + // being cleared + data->bridge = bridge; + bridge->region = region; + assert(data->rc == 1); + + // The region starts with an LRC of 1, due to the local reference to the + // bridge object + regiondata_inc_lrc(region); + regiondata_open(region); + + // This should never fail but might if the given bridge object has + // some object which can't be moved. + if (regiondata_add_object(region, NULL, _PyObject_CAST(bridge))) + { + goto error; + } + + return 0; + +error: + // Cleanup + data->bridge = NULL; + bridge->region = NULL_REGION; + regiondata_dec_rc(region); + return -1; +} + +/* This merges the given region into the local region thereby practically + * dissolving it. + */ +int _PyRegion_Dissolve(Py_region_t region) { + return regiondata_union_merge(region, _Py_LOCAL_REGION); +} + +/* Decrements the reference count of the region. This may deallocate the region. + */ +void _PyRegion_DecRc(Py_region_t region) { + regiondata_dec_rc(region); +} + +/* This removes the pointer from the region to the bridge object. + * + * The bridge object reference is weak, meaning that the RC of the bridge will + * remain unchanged. + */ +void _PyRegion_RemoveBridge(Py_region_t region) { + ASSERT_IS_UNION_ROOT(region); + + // Return for regions without data + if (!HAS_DATA(region)) { + return; + } + + // Clear the name + _Py_region_data *data = (_Py_region_data*)region; + data->bridge = NULL; +} + +Py_ssize_t _PyRegion_GetLrc(Py_region_t region) { + // Sanity Check + ASSERT_IS_UNION_ROOT(region); + + // Return 0 for regions without data + if (!HAS_DATA(region)) { + return 0; + } + + _Py_region_data *data = (_Py_region_data*)region; + return data->lrc; +} + +Py_ssize_t _PyRegion_GetOsc(Py_region_t region) { + // Sanity Check + ASSERT_IS_UNION_ROOT(region); + + // Return 0 for regions without data + if (!HAS_DATA(region)) { + return 0; + } + + _Py_region_data *data = (_Py_region_data*)region; + return data->osc; +} + +/* Returns true, if the given region is marked as dirty + */ +int _PyRegion_IsOpen(Py_region_t region) { + return regiondata_is_open(region); +} + +/* Returns true, if the given region is marked as dirty + */ +int _PyRegion_IsDirty(Py_region_t region) { + return regiondata_is_dirty(region); +} + +int _PyRegion_IsParent(Py_region_t child, Py_region_t parent) { + return regiondata_get_parent(child) == parent; +} + +/* This checks with the region is only held open by the LRC. + * + * Retruns true, if the region will automatically close, once the given + * number (lrc) of local references are dropped. + */ +int _PyRegion_ClosesWithLrc(Py_region_t region, Py_ssize_t lrc) { + return regiondata_closes_after_lrc(region, lrc); +} + +Py_region_t _PyRegion_GetParent(Py_region_t child) { + return regiondata_get_parent(child); +} + +/* This cleans the region by reconstructing it from the bridge object. + * + * FIXME(regions): xFrednet: This could be smarter, by only cleaning + * the region if it's dirty (or a subregion) is dirty. + */ +int _PyRegion_Clean(Py_region_t region) { + if (!HAS_DATA(region)) { + return 0; + } + + _Py_region_data *data = (_Py_region_data *)region; + return regiondata_clean(_PyObject_CAST(data->bridge)); +} + +int _PyRegion_IsBridge(PyObject *obj) { + return _PyRegion_GetBridge(_PyRegion_Get(obj)) == obj; +} + +/* Returns the bridge object belonging to the region of the given object. + */ +PyObject* _PyRegion_GetBridge(Py_region_t region) { + // Regions without data don't have a bridge + if (!HAS_DATA(region)) { + // Return None, since NULL would indicate an exception + Py_RETURN_NONE; + } + + // TODO refactor all uses of this + _Py_region_data *data = (_Py_region_data*)region; + return _PyObject_CAST(data->bridge); +} + +/* Notifys the contianing region that the given object is now immutable. + * This will mark the previously owning region as dirty as the LRC or OSC + * might be invalidated by this move. + * + * This function can fail, if the move closes a parent region. See + * `regiondata_close` for possible failures. + */ +int _PyRegion_SignalImmutable(PyObject *obj) { + Py_region_t region = _PyRegion_Get(obj); + + // Moving an object from a static region is trivial + if (!HAS_DATA(region)) { + return 0; + } + + if (regiondata_is_bridge(region, obj)) { + // Freezing the brigde object might invalidate the OSC of the parent. + // Ideally, we could just unparent the region to prevent the dirty + // mark, but freezing might fail. And if it fails, we would want to + // reconstruct the region and keep the parent relationship. + regiondata_mark_as_dirty(regiondata_get_parent(region)); + } + + // The moved object might have been referenced from the local region + // or reference the bridge of another region. This region change + // therefore invalidates the LRC and OSC of the region. It's marked + // as dirty, and these counts are only reestablished when needed. + regiondata_mark_as_dirty(region); + + return 0; +} + +/* This clears the region from a given object. This should only be done + * when the object is being deallocated. + */ +void _PyRegion_SignalDealloc(PyObject *obj) { + Py_region_t region = _PyRegion_Get(obj); + + // Objects from static regions don't have to be changed. It might + // also be unsafe if the object is shared across threads. + if (!HAS_DATA(region)) { + return; + } + + // Moving the object into a static region, allows the original + // region to be deallocated once te RC hits 0 + _PyRegion_Set(obj, _Py_LOCAL_REGION); +} + +PyRegion_staged_ref_t _PyRegion_StageRef(PyObject *src, PyObject *tgt) { + Py_region_t src_region = _PyRegion_Get(src); + Py_region_t tgt_region = _PyRegion_Get(tgt); + + if (src_region == tgt_region) { + // Intra-region references are always permitted and not tracket + return STAGED_REF_NOP_MERGE; + } + + if (IS_IMMUTABLE_REGION(tgt_region) || IS_COWN_REGION(tgt_region)) { + // References to immutable objects or cowns are always permitted + return STAGED_REF_NOP_MERGE; + } + + if (IS_LOCAL_REGION(src_region)) { + regiondata_inc_lrc(tgt_region); + return (tgt_region | STAGED_REF_LRC_TAG); + } + + return regiondata_stage_object(src_region, src, tgt); +} + +void _PyRegion_ResetStagedRef(PyRegion_staged_ref_t staged_ref) { + staged_ref_reset(staged_ref); +} + +void _PyRegion_CommitStagedRef(PyRegion_staged_ref_t staged_ref) { + staged_ref_commit(staged_ref); +} + +/* Checks if a reference from `src` to `tgt` is allowed and updates the + * internal region state accordingly. + * + * Returns 0 on success. + * + * This is the fast path of `_PyRegion_AddRefs` for single references + */ +int _PyRegion_AddRef(PyObject *src, PyObject *tgt) { + // FIXME(regions): xFrednet: It might be worth to put the fast path into + // the header and allow inlining + + Py_region_t src_region = _PyRegion_Get(src); + Py_region_t tgt_region = _PyRegion_Get(tgt); + + if (src_region == tgt_region) { + // Intra-region references are always permitted and not tracket + return 0; + } + + if (IS_IMMUTABLE_REGION(tgt_region) || IS_COWN_REGION(tgt_region)) { + // References to immutable objects or cowns are always permitted + return 0; + } + + if (IS_LOCAL_REGION(src_region)) { + // References from the local region are allowed, but need to be registered + return regiondata_inc_lrc(tgt_region); + } + + // Attempt to slurp the target object into the source region + return regiondata_add_object(src_region, src, tgt); +} + +/* This informs the regions of the targets about a new incoming local reference. + * + * The `src` argument is only used for error reporting and can be NULL. + */ +static int _add_local_refs(PyObject *src, int tgt_count, PyObject **targets) { + int result = 0; + int arg_i = 0; + + for (arg_i = 0; arg_i < tgt_count; arg_i += 1) { + PyObject* tgt = targets[arg_i]; + result = regiondata_inc_lrc(_PyRegion_Get(tgt)); + + if (result != 0) { + goto error; + } + } + + return 0; + +error: + for (int undo_i = 0; undo_i < arg_i; undo_i += 1) { + PyObject* tgt = targets[undo_i]; + result |= regiondata_dec_lrc(_PyRegion_Get(tgt)); + } + return result; +} + +/* Checks if the references from `src` to the targets are allowed and + * updates the internal region state accordingly. + * + * Returns 0 if all references are allowed. Failure will undo the operation. + * + * The function assumes that the RC of the targets has already been increased. + * Meaning it should be the RC value the value will have, if the operation + * succeeds. + */ +int _PyRegion_AddRefs(PyObject *src, int argc, ...) { + va_list args; + va_start(args, argc); + + assert(argc <= _PyRegion_MAX_ARG_COUNT); + + // Objects which need to be processed further + PyObject *batch[_PyRegion_MAX_ARG_COUNT]; + int batch_size = 0; + + Py_region_t src_region = _PyRegion_Get(src); + for (int arg_i = 0; arg_i < argc; arg_i += 1) { + PyObject* tgt = va_arg(args, PyObject*); + Py_region_t tgt_region = _PyRegion_Get(tgt); + + if (src_region == tgt_region) { + // Intra-region references are always permitted and not tracket + continue; + } + + if (IS_IMMUTABLE_REGION(tgt_region) || IS_COWN_REGION(tgt_region)) { + // References to immutable objects or cowns are always permitted + continue; + } + + // Save the arguments, to be added as a batch + batch[batch_size] = tgt; + batch_size += 1; + } + va_end(args); + + // Return if all references have been trivial + if (batch_size == 0) { + return 0; + } + + if (IS_LOCAL_REGION(src_region)) { + return _add_local_refs(src, batch_size, batch); + } + + // Stage the references to be addeds + PyRegion_staged_ref_t staged_ref = regiondata_stage_objects(src_region, src, batch_size, batch, NULL); + if (staged_ref == PyRegion_staged_ref_ERR) { + return -1; + } + + // Should always succeed + _PyRegion_CommitStagedRef(staged_ref); + return 0; +} + +/* Removes the reference from `src` to `tgt` and updates the internal state of + * the regions. + * + * Returns 0 on success. + */ +int _PyRegion_RemoveRef(PyObject *src, PyObject *tgt) { + if (tgt == NULL) { + return 0; + } + + Py_region_t src_region = _PyRegion_Get(src); + Py_region_t tgt_region = _PyRegion_Get(tgt); + + if (src_region == tgt_region) { + // Intra-region references are always permitted and not tracket + return 0; + } + + if (IS_IMMUTABLE_REGION(tgt_region) || IS_COWN_REGION(tgt_region)) { + // References to immutable objects or cowns are always permitted + return 0; + } + + // Mark the target region as dirty, if the source wasn't passed in. + // This can sadly happen with some old dictionary APIs which don't + // include the dict object + if (src == NULL) { + regiondata_mark_as_dirty(tgt_region); + return 0; + } + + if (IS_LOCAL_REGION(src_region)) { + // Decrease the target region LRC since this reference came from + // the local region + return regiondata_dec_lrc(tgt_region); + } + + if (regiondata_is_bridge(tgt_region, tgt) + && regiondata_get_parent(tgt_region) == src_region + ) { + // The removed reference was the owning references. The target region + // gets unparented and is now free. + return regiondata_set_parent(tgt_region, NULL_REGION); + } else { + // The reference came from `src` to `tgt` while the target region + // already had a parent. This is not allowed but can happend in + // unaware code. The two regions therefore have to be marked as dirty + assert(regiondata_is_dirty(src_region)); + assert(regiondata_is_dirty(tgt_region)); + + // The two regions are marked as dirty. This is an additional safety net + // for builds without asserts. + regiondata_mark_as_dirty(src_region); + regiondata_mark_as_dirty(tgt_region); + + // Still return 0, since the reference could be should be removed. + return 0; + } +} + +int _PyRegion_AddLocalRef(PyObject *tgt) { + return regiondata_inc_lrc(_PyRegion_Get(tgt)); +} + +int _PyRegion_AddLocalRefs(int argc, ...) { + va_list args; + va_start(args, argc); + + assert(argc <= _PyRegion_MAX_ARG_COUNT); + + // Objects which need to be processed further + PyObject *list[_PyRegion_MAX_ARG_COUNT]; + int list_size = 0; + + for (int arg_i = 0; arg_i < argc; arg_i += 1) { + PyObject* tgt = va_arg(args, PyObject*); + + if (!HAS_DATA(_PyRegion_Get(tgt))) { + continue; + } + + // Save the arguments, to be added as a batch + list[list_size] = tgt; + list_size += 1; + } + va_end(args); + + // Return if all references have been trivial + if (list_size == 0) { + return 0; + } + + return _add_local_refs(NULL, list_size, list); +} + +int _PyRegion_RemoveLocalRef(PyObject *tgt) { + return regiondata_dec_lrc(_PyRegion_Get(tgt)); +} + +void _PyRegion_HackDirtyForPrototype(Py_region_t region) { + regiondata_mark_as_dirty(region); +} + +// TODO(regions): xFrednet: Cowns +// Cowns are weird, Regions and RegionObjects are clearly split which allows the +// object to be defined in the regions module. What is the case for cowns? +// When do regions need to know about cowns? +// - Mark a new object as a cown (Move it into the cown region) +// - Inform a cown parent that a region is closed +// - Region has to know that it's owned +// Idea: Have a Cown struct, which is not bound to the CownObject of BoC. This +// object has a callback for `close,open,merge`? +// This should allow the creation of the CownObject but also other magic +// I think this is good (famous last words) +// TODO(regions): xFrednet: Write Barrier in: Bytecode +// TODO(regions): xFrednet: Write Barrier in: Dictionary +// TODO(regions): xFrednet: Dirty on C code +// TODO(regions): xFrednet: Track Weak Reference in LRC +// TODO(regions): xFrednet: Weak Reference into regions +// TODO(regions): xFrednet: Merging a region into the local region should open +// subregions, if the merge didn't happend for error handling +// (Make sure subregions are always at the start of the region CG list) +// (This might need a custom list, since bridges are currently part of +// their regions list) +// (Can this opening be done by checking if the parent is local?) +// TODO(regions): xFrednet: Add GC operating in individual regions diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ee6cff9efc7856..4e2bcf92a163d2 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -218,6 +218,7 @@ def clean_lines(text): Modules/main.c Py_BUILD_CORE 1 Modules/mathmodule.c Py_BUILD_CORE 1 Modules/posixmodule.c Py_BUILD_CORE 1 +Modules/regionsmodule.c Py_BUILD_CORE 1 Modules/sha256module.c Py_BUILD_CORE 1 Modules/sha512module.c Py_BUILD_CORE 1 Modules/signalmodule.c Py_BUILD_CORE 1 diff --git a/configure b/configure index 1ded6a62e0fafd..04dcd8f2939b84 100755 --- a/configure +++ b/configure @@ -795,6 +795,8 @@ MODULE__REMOTE_DEBUGGING_FALSE MODULE__REMOTE_DEBUGGING_TRUE MODULE__RANDOM_FALSE MODULE__RANDOM_TRUE +MODULE_REGIONS_FALSE +MODULE_REGIONS_TRUE MODULE__QUEUE_FALSE MODULE__QUEUE_TRUE MODULE__POSIXSUBPROCESS_FALSE @@ -1095,6 +1097,7 @@ with_static_libpython enable_profiling enable_gil with_pydebug +with_ownership_invariant with_trace_refs enable_pystats with_assertions @@ -1882,6 +1885,9 @@ Optional Packages: do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes) --with-pydebug build with Py_DEBUG defined (default is no) + --with-ownership-invariant + enable ownership invariant for debugging purpose + (default is no) --with-trace-refs enable tracing references for debugging purpose (default is no) --with-assertions build with C assertions enabled (default is no) @@ -8296,6 +8302,31 @@ esac fi +# Check for --with-ownership-invariant +# --with-ownership-invariant +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-ownership-invariant" >&5 +printf %s "checking for --with-ownership-invariant... " >&6; } + +# Check whether --with-ownership-invariant was given. +if test ${with_ownership_invariant+y} +then : + withval=$with_ownership_invariant; +else case e in #( + e) with_ownership_invariant=no + ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_ownership_invariant" >&5 +printf "%s\n" "$with_ownership_invariant" >&6; } + +if test "$with_ownership_invariant" = "yes" +then + +printf "%s\n" "#define Py_OWNERSHIP_INVARIANT 1" >>confdefs.h + +fi + # Check for --with-trace-refs # --with-trace-refs { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-trace-refs" >&5 @@ -31558,6 +31589,28 @@ then : +fi + + + if test "$py_cv_module_regions" != "n/a" +then : + py_cv_module_regions=yes +fi + if test "$py_cv_module_regions" = yes; then + MODULE_REGIONS_TRUE= + MODULE_REGIONS_FALSE='#' +else + MODULE_REGIONS_TRUE='#' + MODULE_REGIONS_FALSE= +fi + + as_fn_append MODULE_BLOCK "MODULE_REGIONS_STATE=$py_cv_module_regions$as_nl" + if test "x$py_cv_module_regions" = xyes +then : + + + + fi @@ -34467,6 +34520,10 @@ if test -z "${MODULE__QUEUE_TRUE}" && test -z "${MODULE__QUEUE_FALSE}"; then as_fn_error $? "conditional \"MODULE__QUEUE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE_REGIONS_TRUE}" && test -z "${MODULE_REGIONS_FALSE}"; then + as_fn_error $? "conditional \"MODULE_REGIONS\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__RANDOM_TRUE}" && test -z "${MODULE__RANDOM_FALSE}"; then as_fn_error $? "conditional \"MODULE__RANDOM\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 3261a453d4e608..e5104ebdadcb00 100644 --- a/configure.ac +++ b/configure.ac @@ -1746,6 +1746,21 @@ else AC_MSG_RESULT([no]); Py_DEBUG='false' fi], [AC_MSG_RESULT([no])]) +# Check for --with-ownership-invariant +# --with-ownership-invariant +AC_MSG_CHECKING([for --with-ownership-invariant]) +AC_ARG_WITH([ownership-invariant], + [AS_HELP_STRING([--with-ownership-invariant], [enable ownership invariant for debugging purpose (default is no)])], + [], [with_ownership_invariant=no] +) +AC_MSG_RESULT([$with_ownership_invariant]) + +if test "$with_ownership_invariant" = "yes" +then + AC_DEFINE([Py_OWNERSHIP_INVARIANT], [1], + [Define if you want to enable ownership invariant for debugging purpose]) +fi + # Check for --with-trace-refs # --with-trace-refs AC_MSG_CHECKING([for --with-trace-refs]) @@ -7908,6 +7923,7 @@ PY_STDLIB_MOD_SIMPLE([_lsprof]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) +PY_STDLIB_MOD_SIMPLE([regions]) PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([_remote_debugging]) PY_STDLIB_MOD_SIMPLE([select]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 65a2c55217c258..e0423873c8c89a 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1714,6 +1714,9 @@ /* Define to 1 if you have the perf trampoline. */ #undef PY_HAVE_PERF_TRAMPOLINE +/* Define if you want to enable ownership invariant for debugging purpose */ +#undef Py_OWNERSHIP_INVARIANT + /* Define to 1 to build the sqlite module with loadable extensions support. */ #undef PY_SQLITE_ENABLE_LOAD_EXTENSION