diff --git a/CMakeLists.txt b/CMakeLists.txt index 79828f6e..c3e857c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ include(GNUInstallDirs) # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests? option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF) set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "") -set(CPPINTEROP_GIT_TAG "c8126249dd18363000b89333b34313ca2a83fb82" CACHE STRING "") +set(CPPINTEROP_GIT_TAG "c57d253e7cba94cc0976a9e36547a960f0fd2f83" CACHE STRING "") set(CPPINTEROP_SOURCE_DIR "" CACHE PATH "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source") diff --git a/python/cppjit/__init__.py b/python/cppjit/__init__.py index 8df1b926..dfb5a189 100644 --- a/python/cppjit/__init__.py +++ b/python/cppjit/__init__.py @@ -48,6 +48,7 @@ "add_library_path", # add a path to search for libraries "add_autoload_map", # explicitly include an autoload map "set_debug", # enable/disable debug output + "use_memory_analyzer", # enable/disable the memory ownership analyzer ] import ctypes @@ -397,6 +398,11 @@ def set_debug(enable=True): gbl.Cpp.EnableDebugOutput(enable) +def use_memory_analyzer(enable=True): + """Enable/disable the memory ownership analyzer.""" + _backend.UseMemoryAnalyzer(enable) + + def _get_name(tt): if isinstance(tt, str): return tt diff --git a/src/cpyrt/CPPClassMethod.cxx b/src/cpyrt/CPPClassMethod.cxx index bbdc1022..2ff21efe 100644 --- a/src/cpyrt/CPPClassMethod.cxx +++ b/src/cpyrt/CPPClassMethod.cxx @@ -50,6 +50,24 @@ PyObject* cpyrt::CPPClassMethod::Call(CPPInstance*& self, cpyrt_PyArgs_t args, return this->Execute(nullptr, 0, ctxt); } +//---------------------------------------------------------------------------- +Py_ssize_t cpyrt::CPPClassMethod::GetDeallocArgShift(bool isBound, + PyObject* const* args, + Py_ssize_t nargs) { + // a static method takes no object argument, so a bound call drops self + if (isBound) + return 0; + + // Call function above drops a leading instance that the call does not need, + // and then args[0] belongs to no parameter. The check below repeats the one + // in Call(), so a change to either one belongs in both. + if (nargs > 0 && CPPInstance_Check(args[0]) && fArgsRequired <= nargs - 1 && + interop::IsSubclass(((CPPInstance*)args[0])->ObjectIsA(), GetScope())) + return -1; + + return 0; +} + //---------------------------------------------------------------------------- PyObject* cpyrt::CPPClassMethod::GetTypeName() { PyObject* cppname = diff --git a/src/cpyrt/CPPClassMethod.h b/src/cpyrt/CPPClassMethod.h index 4ffe761c..bd79b043 100644 --- a/src/cpyrt/CPPClassMethod.h +++ b/src/cpyrt/CPPClassMethod.h @@ -14,6 +14,11 @@ class CPPClassMethod : public CPPMethod { PyObject* GetTypeName() override; public: + Py_ssize_t GetDeallocArgShift(bool isBound, PyObject* const* args, + Py_ssize_t nargs) override; + // a static method has no object argument. + Py_ssize_t GetDeallocParamShift() override { return 0; } + PyCallable* Clone() override { return new CPPClassMethod(*this); } PyObject* Call(CPPInstance*& self, cpyrt_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr) override; diff --git a/src/cpyrt/CPPFunction.h b/src/cpyrt/CPPFunction.h index 9e356c86..4e3b27e5 100644 --- a/src/cpyrt/CPPFunction.h +++ b/src/cpyrt/CPPFunction.h @@ -15,6 +15,11 @@ class CPPFunction : public CPPMethod { PyObject* GetTypeName() override; public: + // A free function takes its object as a plain first parameter, so it has no + // implicit this slot. The shift of a bound call stays the one of CPPMethod, + // as AdjustSelf() puts self in that first parameter. + Py_ssize_t GetDeallocParamShift() override { return 0; } + PyCallable* Clone() override { return new CPPFunction(*this); } PyObject* Call(CPPInstance*& self, cpyrt_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr) override; diff --git a/src/cpyrt/CPPInstance.cxx b/src/cpyrt/CPPInstance.cxx index ec882a53..fa5c2aa3 100644 --- a/src/cpyrt/CPPInstance.cxx +++ b/src/cpyrt/CPPInstance.cxx @@ -17,6 +17,7 @@ using namespace cppjit; // Standard #include +#include #include //- data _____________________________________________________________________ @@ -227,7 +228,17 @@ void cpyrt::op_dealloc_nofree(CPPInstance* pyobj) { if (pyobj->fFlags & CPPInstance::kIsValue) { interop::CallDestructor(klass, cppobj); interop::Deallocate(klass, cppobj); - } else + } else if (pyobj->fFlags & CPPInstance::kIsMalloc) + std::free(cppobj); + else if (pyobj->fFlags & CPPInstance::kIsNoConstruct) { + if (pyobj->fFlags & CPPInstance::kIsArrayAlloc) + ::operator delete[](cppobj); + else + ::operator delete(cppobj); + } else if (pyobj->fFlags & CPPInstance::kIsArrayAlloc) + interop::Destruct(klass, cppobj, 1); + // Default case: just kIsOwner set in all of memory-ownership flags + else interop::Destruct(klass, cppobj); } cppobj = nullptr; @@ -938,10 +949,30 @@ static int op_setownership(CPPInstance* pyobj, PyObject* value, void*) { return 0; } +// Added for testing purposes +//----------------------------------------------------------------------------- +static PyObject* op_get_array_alloc(CPPInstance* pyobj, void*) { + return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsArrayAlloc)); +} +//----------------------------------------------------------------------------- +static PyObject* op_get_no_construct(CPPInstance* pyobj, void*) { + return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsNoConstruct)); +} +//----------------------------------------------------------------------------- +static PyObject* op_get_malloc(CPPInstance* pyobj, void*) { + return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsMalloc)); +} //----------------------------------------------------------------------------- static PyGetSetDef op_getset[] = { {(char*)"__python_owns__", (getter)op_getownership, (setter)op_setownership, (char*)"If true, python manages the life time of this object", nullptr}, + {(char*)"__is_array_alloc__", (getter)op_get_array_alloc, nullptr, + (char*)"If true, the object was allocated with new[]/operator new[]", + nullptr}, + {(char*)"__is_no_construct__", (getter)op_get_no_construct, nullptr, + (char*)"If true, the memory is raw: no constructor was run", nullptr}, + {(char*)"__is_malloc__", (getter)op_get_malloc, nullptr, + (char*)"If true, the object was allocated with malloc", nullptr}, {(char*)nullptr, nullptr, nullptr, nullptr, nullptr}}; //= cpyrt type number stubs to allow dynamic overrides ===================== diff --git a/src/cpyrt/CPPInstance.h b/src/cpyrt/CPPInstance.h index c94ecfa1..00d6d7cc 100644 --- a/src/cpyrt/CPPInstance.h +++ b/src/cpyrt/CPPInstance.h @@ -29,20 +29,24 @@ class CPPInstance { public: enum EFlags { kDefault = 0x0000, - kNoWrapConv = 0x0001, // use type as-is (eg. no smart ptr wrap) - kIsOwner = 0x0002, // Python instance owns C++ object/memory - kIsExtended = 0x0004, // has extended data - kIsValue = 0x0008, // was created from a by-value return - kIsReference = 0x0010, // represents one indirection - kIsArray = 0x0020, // represents an array of objects - kIsSmartPtr = 0x0040, // is or embeds a smart pointer - kIsPtrPtr = 0x0080, // represents two indirections - kIsRValue = 0x0100, // can be used as an r-value - kIsLValue = 0x0200, // can be used as an l-value - kNoMemReg = 0x0400, // do not register with memory regulator - kIsRegulated = 0x0800, // is registered with memory regulator - kIsActual = 0x1000, // has been downcasted to actual type - kHasLifeLine = 0x2000, // has a life line set + kNoWrapConv = 0x0001, // use type as-is (eg. no smart ptr wrap) + kIsOwner = 0x0002, // Python instance owns C++ object/memory + kIsExtended = 0x0004, // has extended data + kIsValue = 0x0008, // was created from a by-value return + kIsReference = 0x0010, // represents one indirection + kIsArray = 0x0020, // represents an array of objects + kIsSmartPtr = 0x0040, // is or embeds a smart pointer + kIsPtrPtr = 0x0080, // represents two indirections + kIsRValue = 0x0100, // can be used as an r-value + kIsLValue = 0x0200, // can be used as an l-value + kNoMemReg = 0x0400, // do not register with memory regulator + kIsRegulated = 0x0800, // is registered with memory regulator + kIsActual = 0x1000, // has been downcasted to actual type + kHasLifeLine = 0x2000, // has a life line set + kIsArrayAlloc = 0x4000, // represents a heap allocated array of objects + kIsNoConstruct = + 0x8000, // represents constructor is not called in the allocation + kIsMalloc = 0x10000, // is allocated with malloc }; public: // public, as the python C-API works with C structs diff --git a/src/cpyrt/CPPMethod.cxx b/src/cpyrt/CPPMethod.cxx index 0221f836..74a3d9ac 100644 --- a/src/cpyrt/CPPMethod.cxx +++ b/src/cpyrt/CPPMethod.cxx @@ -32,6 +32,7 @@ extern PyObject* gBusException; extern PyObject* gSegvException; extern PyObject* gIllException; extern PyObject* gAbrtException; +extern bool gUseMemoryAnalyzer; } // namespace cppjit::cpyrt //- public helper ------------------------------------------------------------ @@ -85,6 +86,8 @@ inline void cpyrt::CPPMethod::Copy_(const CPPMethod& /* other */) { fExecutor = nullptr; fArgIndices = nullptr; fArgsRequired = -1; + fAllocType.reset(); + fDeallocType.reset(); } //---------------------------------------------------------------------------- @@ -703,6 +706,64 @@ PyObject* cpyrt::CPPMethod::GetArgDefault(int iarg, bool silent) { bool cpyrt::CPPMethod::IsConst() { return interop::IsConstMethod(GetMethod()); } +//---------------------------------------------------------------------------- +interop::AllocType cpyrt::CPPMethod::GetAllocBehaviour() { + if (fAllocType.has_value()) + return *fAllocType; + interop::TCppMethod_t method = GetMethod(); + if (!method) { + fAllocType = interop::AllocType::None; + return interop::AllocType::None; + } + if (!IsPointerType(interop::GetMethodReturnType(method))) { + fAllocType = interop::AllocType::None; + return interop::AllocType::None; + } + interop::AllocType attrResult = interop::IsAllocator(method); + // an attribute always wins, so ask the analyzer when there is no + // relevant attribute + if (attrResult == interop::AllocType::Unknown && gUseMemoryAnalyzer) + attrResult = interop::GetAllocType(method); + fAllocType = attrResult; + return attrResult; +} + +//---------------------------------------------------------------------------- +void cpyrt::CPPMethod::FillDeallocBehaviour() { + fDeallocType.emplace(); + interop::TCppMethod_t method = GetMethod(); + if (!method) + return; + + interop::IsDeallocator(method, *fDeallocType); + if (!gUseMemoryAnalyzer) + return; + + std::vector analyzeRes; + if (!interop::GetDeallocType(method, analyzeRes)) + return; + + // IsDeallocator puts the implicit this in slot 0, GetDeallocType does not, + // so each analyzer answer moves up by one slot for a method + const size_t shift = (size_t)GetDeallocParamShift(); + assert(analyzeRes.size() + shift == fDeallocType->size() && + "the analyzer answer must line up with the attribute slots"); + for (size_t i = 0; i < analyzeRes.size(); i++) { + // an attribute always wins, so fill only the slots there is no attribute + if ((*fDeallocType)[i + shift] == interop::DeallocType::Opaque) + (*fDeallocType)[i + shift] = analyzeRes[i]; + } +} + +//---------------------------------------------------------------------------- +interop::DeallocType cpyrt::CPPMethod::GetDeallocBehaviour(size_t index) { + if (!fDeallocType) + FillDeallocBehaviour(); + if (index < fDeallocType->size()) + return (*fDeallocType)[index]; + return interop::DeallocType::Opaque; +} + //---------------------------------------------------------------------------- PyObject* cpyrt::CPPMethod::GetScopeProxy() { // Get or build the scope of this method. diff --git a/src/cpyrt/CPPMethod.h b/src/cpyrt/CPPMethod.h index 4fdd792d..99b56b20 100644 --- a/src/cpyrt/CPPMethod.h +++ b/src/cpyrt/CPPMethod.h @@ -5,6 +5,7 @@ #include "PyCallable.h" // Standard +#include #include #include #include @@ -62,6 +63,17 @@ class CPPMethod : public PyCallable { PyObject* GetCoVarNames() override; PyObject* GetArgDefault(int iarg, bool silent = true) override; bool IsConst() override; + interop::AllocType GetAllocBehaviour() override; + interop::DeallocType GetDeallocBehaviour(size_t index) override; + // A bound call passes self on its own, so args[0] lines up with the second + // slot. A direct call goes through the vector call, which skips the + // descriptor, and then self sits in args[0] itself. + Py_ssize_t GetDeallocArgShift(bool isBound, PyObject* const* /* args */, + Py_ssize_t /* nargs */) override { + return isBound ? 1 : 0; + } + // the first slot belongs to the implicit object argument of the method + Py_ssize_t GetDeallocParamShift() override { return 1; } PyObject* GetScopeProxy() override; interop::TCppFuncAddr_t GetFunctionAddress() override; @@ -106,6 +118,8 @@ class CPPMethod : public PyCallable { void SetPyError_(PyObject* msg); + void FillDeallocBehaviour(); + private: // representation interop::TCppMethod_t fMethod; @@ -121,6 +135,8 @@ class CPPMethod : public PyCallable { protected: // cached value that doubles as initialized flag (uninitialized if -1) int fArgsRequired; + std::optional fAllocType; + std::optional> fDeallocType; }; } // namespace cppjit::cpyrt diff --git a/src/cpyrt/CPPOverload.cxx b/src/cpyrt/CPPOverload.cxx index 96016205..fd43dcce 100644 --- a/src/cpyrt/CPPOverload.cxx +++ b/src/cpyrt/CPPOverload.cxx @@ -5,6 +5,7 @@ using namespace cppjit; #include "CPPInstance.h" #include "CPPOverload.h" #include "CallContext.h" +#include "LowLevelViews.h" #include "PyStrings.h" #include "Utility.h" #include "structmember.h" // from Python @@ -142,14 +143,27 @@ static inline void ResetCallState(CPPInstance* descr_self, // helper to factor out return logic of mp_call / mp_vectorcall static inline PyObject* HandleReturn(CPPOverload* pymeth, CPPInstance* im_self, - PyObject* result) { + PyObject* result, PyCallable* method, + PyObject* const* args, size_t nargsf, + PyObject* kwds) { // special case for python exceptions, propagated through C++ layer if (result) { CPPInstance* cppres = (CPPInstance*)(CPPInstance_Check(result) ? result : nullptr); + interop::AllocType AT = method->GetAllocBehaviour(); + // the overloads of a group share fMethodInfo, so the answer of this one + // stays in a local instead of going into the shared flags + const bool allocates = AT != interop::AllocType::None && + AT != interop::AllocType::Null && + AT != interop::AllocType::Unknown; + // if this method creates new objects, always take ownership - if (IsCreator(pymeth->fMethodInfo->fFlags)) { + if (allocates || IsCreator(pymeth->fMethodInfo->fFlags)) { + + // Note: for CPPInstance returning functions, if IsCreator(..) returns + // true it will be assumed allocation is done with `new`, and for + // LowLevelView objects, it will be assumed allocator is `new[]` // either be a constructor with a fresh object proxy self ... if (IsConstructor(pymeth->fMethodInfo->fFlags)) { @@ -158,9 +172,97 @@ static inline PyObject* HandleReturn(CPPOverload* pymeth, CPPInstance* im_self, } // ... or be a regular method with an object proxy return value - else if (cppres) + else if (cppres) { cppres->PythonOwns(); + // After giving ownership, set proper flags to indicate allocation + // method/func + switch (AT) { + case interop::AllocType::Malloc: + cppres->fFlags |= CPPInstance::kIsMalloc; + break; + case interop::AllocType::NewArr: + cppres->fFlags |= CPPInstance::kIsArrayAlloc; + break; + case interop::AllocType::OperatorNew: + cppres->fFlags |= CPPInstance::kIsNoConstruct; + break; + case interop::AllocType::OperatorNewArr: + cppres->fFlags |= CPPInstance::kIsArrayAlloc; + cppres->fFlags |= CPPInstance::kIsNoConstruct; + break; + default: + break; + } + } else if (LowLevelView_Check(result)) { + LowLevelView* LL = (LowLevelView*)result; + auto& flags = (intptr_t&)LL->fBufInfo.internal; + flags |= LowLevelView::kIsOwner; + switch (AT) { + case interop::AllocType::Malloc: + flags &= ~LowLevelView::kIsCppArray; + break; + case interop::AllocType::New: + case interop::AllocType::OperatorNew: + flags |= LowLevelView::kIsNew; + flags &= ~LowLevelView::kIsCppArray; + break; + // ll_new assumes LL is allocated with newarr default, so no need to + // touch + default: + break; + } + } + } + + // an argument that the called function releases belongs to C++ from now + // on, else Python frees it a second time + const Py_ssize_t nargs = cpyrt_PyArgs_GET_SIZE(args, nargsf); + const Py_ssize_t numKwds = + kwds && PyTuple_CheckExact(kwds) ? PyTuple_GET_SIZE(kwds) : 0; + // the keyword values follow the positional ones in call order, so their + // names are what tell which parameter each one belongs to + PyObject* argnames = numKwds ? method->GetSignatureNames() : nullptr; + + const bool isBound = pymeth->fSelf != nullptr; + // a place in the array becomes a slot with this shift + const Py_ssize_t argShift = + method->GetDeallocArgShift(isBound, args, nargs); + // a parameter index becomes a slot with this one + const Py_ssize_t paramShift = method->GetDeallocParamShift(); + // the array holds the positional values first, then the keyword ones + for (Py_ssize_t i = 0; i < nargs + numKwds; i++) { + if (!CPPInstance_Check(args[i]) && !LowLevelView_Check(args[i])) + continue; + + Py_ssize_t slot = -1; // no slot of its own + if (i < nargs) + slot = i + argShift; + else { + // kwds runs parallel to the keyword part of the array + // the place of the name in the signature is the parameter index + for (Py_ssize_t iarg = 0; iarg < PyTuple_GET_SIZE(argnames); iarg++) { + if (PyUnicode_Compare(PyTuple_GET_ITEM(argnames, iarg), + PyTuple_GET_ITEM(kwds, i - nargs)) == 0) { + slot = iarg + paramShift; + break; + } + } + } + // the callee drops this argument, or no name of it matches a parameter + if (slot < 0) + continue; + + const interop::DeallocType DT = method->GetDeallocBehaviour((size_t)slot); + if (DT == interop::DeallocType::None || + DT == interop::DeallocType::Opaque) + continue; + if (CPPInstance_Check(args[i])) + ((CPPInstance*)args[i])->CppOwns(); + else + (intptr_t&)((LowLevelView*)args[i])->fBufInfo.internal &= + ~LowLevelView::kIsOwner; } + Py_XDECREF(argnames); // if this new object falls inside self, make sure its lifetime is proper if (!(pymeth->fMethodInfo->fFlags & CallContext::kNeverLifeLine)) { @@ -576,7 +678,8 @@ static PyObject* mp_vectorcall(CPPOverload* pymeth, PyObject* const* args, if (!NoImplicit(&ctxt)) ctxt.fFlags |= CallContext::kAllowImplicit; // no two rounds needed PyObject* result = methods[0]->Call(im_self, args, nargsf, kwds, &ctxt); - return HandleReturn(pymeth, im_self, result); + return HandleReturn(pymeth, im_self, result, methods[0], args, nargsf, + kwds); } // otherwise, handle overloading @@ -599,7 +702,8 @@ static PyObject* mp_vectorcall(CPPOverload* pymeth, PyObject* const* args, ctxt.fFlags |= CallContext::kAllowImplicit; PyObject* result = memoized_pc->Call(im_self, args, nargsf, kwds, &ctxt); if (result) - return HandleReturn(pymeth, im_self, result); + return HandleReturn(pymeth, im_self, result, memoized_pc, args, nargsf, + kwds); // fall through: python is dynamic, and so, the hashing isn't infallible ctxt.fFlags &= ~(CallContext::kAllowImplicit | CallContext::kPyException | @@ -656,7 +760,8 @@ static PyObject* mp_vectorcall(CPPOverload* pymeth, PyObject* const* args, } } - return HandleReturn(pymeth, im_self, result); + return HandleReturn(pymeth, im_self, result, methods[i], args, nargsf, + kwds); } // else failure .. diff --git a/src/cpyrt/LowLevelViews.cxx b/src/cpyrt/LowLevelViews.cxx index d4d81fa5..8d390892 100644 --- a/src/cpyrt/LowLevelViews.cxx +++ b/src/cpyrt/LowLevelViews.cxx @@ -11,6 +11,7 @@ using namespace cppjit; #include #include #include +#include #include //= memoryview-like object =================================================== @@ -66,9 +67,11 @@ static void ll_dealloc(cpyrt::LowLevelView* pyobj) { PyMem_Free(pyobj->fBufInfo.strides); if ((intptr_t)pyobj->fBufInfo.internal & cpyrt::LowLevelView::kIsOwner) { if ((intptr_t)pyobj->fBufInfo.internal & cpyrt::LowLevelView::kIsCppArray) - delete[] pyobj->fBuf; + ::operator delete[](pyobj->get_buf()); + else if ((intptr_t)pyobj->fBufInfo.internal & cpyrt::LowLevelView::kIsNew) + ::operator delete(pyobj->get_buf()); else - free(pyobj->fBuf); + free(pyobj->get_buf()); } if (pyobj->fElemCnv != pyobj->fConverter && pyobj->fElemCnv && @@ -107,6 +110,7 @@ static void ll_dealloc(cpyrt::LowLevelView* pyobj) { // clang-format off CPYRT_LL_FLAG_GETSET(ownership, cpyrt::LowLevelView::kIsOwner, __python_owns__) CPYRT_LL_FLAG_GETSET(cpparray, cpyrt::LowLevelView::kIsCppArray, __cpp_array__) +CPYRT_LL_FLAG_GETSET(isnew, cpyrt::LowLevelView::kIsNew, __is_new__) // clang-format on //--------------------------------------------------------------------------- @@ -916,6 +920,8 @@ static PyGetSetDef ll_getset[] = { {(char*)"format", (getter)ll_typecode, nullptr, nullptr, nullptr}, {(char*)"typecode", (getter)ll_typecode, nullptr, nullptr, nullptr}, {(char*)"shape", (getter)ll_shape, (setter)ll_setshape, nullptr, nullptr}, + {(char*)"__is_new__", (getter)ll_getisnew, (setter)ll_setisnew, + (char*)"If true, this buffer is allocated with new", nullptr}, {(char*)nullptr, nullptr, nullptr, nullptr, nullptr}}; namespace cppjit::cpyrt { diff --git a/src/cpyrt/LowLevelViews.h b/src/cpyrt/LowLevelViews.h index 80e7f02b..b3bb2e7d 100644 --- a/src/cpyrt/LowLevelViews.h +++ b/src/cpyrt/LowLevelViews.h @@ -19,8 +19,9 @@ class LowLevelView { kDefault = 0x0000, kIsCppArray = 0x0001, // allocated with new[] kIsFixed = 0x0002, // fixed size array (assumed flat) - kIsOwner = 0x0004 - }; // Python owns + kIsOwner = 0x0004, // Python owns + kIsNew = 0x0008 // memory is allocated with new + }; public: PyObject_HEAD Py_buffer fBufInfo; diff --git a/src/cpyrt/PyCallable.h b/src/cpyrt/PyCallable.h index 4b79ab00..3cc1e7ea 100644 --- a/src/cpyrt/PyCallable.h +++ b/src/cpyrt/PyCallable.h @@ -37,6 +37,22 @@ class PyCallable { virtual PyObject* GetCoVarNames() = 0; virtual PyObject* GetArgDefault(int /* iarg */, bool silent = true) = 0; virtual bool IsConst() { return false; } + virtual interop::AllocType GetAllocBehaviour() { + return interop::AllocType::None; + } + virtual interop::DeallocType GetDeallocBehaviour(size_t /* index */) { + return interop::DeallocType::Opaque; + } + // shift that turns a place in the args array into a GetDeallocBehaviour() + // slot, which the shape of the call decides + virtual Py_ssize_t GetDeallocArgShift(bool /* isBound */, + PyObject* const* /* args */, + Py_ssize_t /* nargs */) { + return 0; + } + // shift that turns a declared parameter index into a GetDeallocBehaviour() + // slot, which the declaration decides + virtual Py_ssize_t GetDeallocParamShift() { return 0; } virtual PyObject* GetScopeProxy() = 0; virtual interop::TCppFuncAddr_t GetFunctionAddress() = 0; diff --git a/src/cpyrt/cpyrtModule.cxx b/src/cpyrt/cpyrtModule.cxx index 915ceb74..c89ddbc3 100644 --- a/src/cpyrt/cpyrtModule.cxx +++ b/src/cpyrt/cpyrtModule.cxx @@ -240,6 +240,7 @@ PyObject* gAbrtException = nullptr; std::unordered_set gPinnedTypes; std::ostringstream gCapturedError; std::streambuf* gOldErrorBuffer = nullptr; +bool gUseMemoryAnalyzer = false; std::unordered_map>& pythonizations() { static std::unordered_map> pyzMap; @@ -585,14 +586,21 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) { // ownership is taken over as needed, again following the principle of // "least surprise" as most likely only the cast object will be retained bool owns = arg0_pyobj->fFlags & CPPInstance::kIsOwner; + // allocation flags travels with the ownership + const unsigned allocFlags = + arg0_pyobj->fFlags & + (CPPInstance::kIsMalloc | CPPInstance::kIsArrayAlloc | + CPPInstance::kIsNoConstruct); if (!isPython) { // ordinary C++ class PyObject* pyobj = BindCppObjectNoCast( interop::TCppObject_t((void*)((intptr_t)address.data + offset)), cast_type, owns ? CPPInstance::kIsOwner : 0); - if (owns && pyobj) + if (owns && pyobj) { arg0_pyobj->CppOwns(); + ((CPPInstance*)pyobj)->fFlags |= allocFlags; + } return pyobj; } else { @@ -624,6 +632,7 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) { if (pyobj && owns) { arg0_pyobj->CppOwns(); ((CPPInstance*)pyobj)->PythonOwns(); + ((CPPInstance*)pyobj)->fFlags |= allocFlags; } return pyobj; @@ -907,6 +916,19 @@ static PyObject* EndCaptureStderr(PyObject*, PyObject*) { return Py_BuildValue("s", capturedError.c_str()); } + +//---------------------------------------------------------------------------- +static PyObject* UseMemoryAnalyzer(PyObject*, PyObject* args) { + // off by default, because the analyzer reads function bodies and only + // guesses the ownership that an attribute would state + int enable = 0; + if (!PyArg_ParseTuple(args, const_cast("p"), &enable)) + return nullptr; + + gUseMemoryAnalyzer = enable; + + Py_RETURN_NONE; +} } // unnamed namespace //- data ----------------------------------------------------------------------- @@ -965,6 +987,8 @@ static PyMethodDef gcpyrtMethods[] = { METH_NOARGS, (char*)"Begin capturing stderr to a in memory buffer."}, {(char*)"_end_capture_stderr", (PyCFunction)EndCaptureStderr, METH_NOARGS, (char*)"End capturing stderr and returns the captured buffer."}, + {(char*)"UseMemoryAnalyzer", (PyCFunction)UseMemoryAnalyzer, METH_VARARGS, + (char*)"Enable/disable the memory ownership analyzer."}, {nullptr, nullptr, 0, nullptr}}; struct module_state { diff --git a/src/interop/cppjit_interop.h b/src/interop/cppjit_interop.h index 59c1a95b..b0a34c0a 100644 --- a/src/interop/cppjit_interop.h +++ b/src/interop/cppjit_interop.h @@ -44,6 +44,8 @@ typedef Cpp::FuncRef TCppMethod_t; typedef Cpp::InterpRef TInterp_t; typedef size_t TCppIndex_t; typedef void* TCppFuncAddr_t; +typedef Cpp::AllocType AllocType; +typedef Cpp::DeallocType DeallocType; // direct interpreter access ------------------------------------------------- RPY_EXPORTED @@ -132,7 +134,7 @@ void Deallocate(TCppScope_t scope, TCppObject_t instance); RPY_EXPORTED TCppObject_t Construct(TCppScope_t scope, void* arena = nullptr); RPY_EXPORTED -void Destruct(TCppScope_t scope, TCppObject_t instance); +void Destruct(TCppScope_t scope, TCppObject_t instance, size_t count = 0); // method/function dispatching ----------------------------------------------- RPY_EXPORTED @@ -297,6 +299,14 @@ RPY_EXPORTED std::string GetDoxygenComment(TCppScope_t scope, bool strip_markers = true); RPY_EXPORTED bool IsConstMethod(TCppMethod_t); +RPY_EXPORTED +AllocType IsAllocator(TCppMethod_t); +RPY_EXPORTED +AllocType GetAllocType(TCppMethod_t); +RPY_EXPORTED +bool IsDeallocator(TCppMethod_t, std::vector& VPP); +RPY_EXPORTED +bool GetDeallocType(TCppMethod_t, std::vector& VPP); // Templated method/function reflection information // ------------------------------------ RPY_EXPORTED diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index d26697c8..76244a51 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -845,10 +845,11 @@ interop::TCppObject_t interop::Construct(TCppScope_t scope, return Cpp::Construct(scope, arena, /*count=*/1); } -void interop::Destruct(TCppScope_t scope, TCppObject_t instance) { +void interop::Destruct(TCppScope_t scope, TCppObject_t instance, + size_t count /*=0*/) { std::lock_guard Lock( InterOpMutex); // TODO: this shouldn't locks the JIT call - Cpp::Destruct(instance, scope, true, /*count=*/0); + Cpp::Destruct(instance, scope, true, count); } static inline bool copy_args(Parameter* args, size_t nargs, void** vargs) { @@ -1250,6 +1251,28 @@ interop::TCppType_t interop::GetMethodReturnType(TCppMethod_t method) { return Cpp::GetFunctionReturnType(method); } +bool interop::IsDeallocator(TCppMethod_t method, + std::vector& VPP) { + std::lock_guard Lock(InterOpMutex); + return Cpp::IsDeallocator(method, VPP); +} + +bool interop::GetDeallocType(TCppMethod_t method, + std::vector& VPP) { + std::lock_guard Lock(InterOpMutex); + return Cpp::GetDeallocType(method, VPP); +} + +interop::AllocType interop::IsAllocator(TCppMethod_t method) { + std::lock_guard Lock(InterOpMutex); + return Cpp::IsAllocator(method); +} + +interop::AllocType interop::GetAllocType(TCppMethod_t method) { + std::lock_guard Lock(InterOpMutex); + return Cpp::GetAllocType(method); +} + std::string interop::GetMethodReturnTypeAsString(TCppMethod_t method) { std::lock_guard Lock(InterOpMutex); return Cpp::GetTypeAsString( diff --git a/test/Makefile b/test/Makefile index 7c891304..60070aea 100644 --- a/test/Makefile +++ b/test/Makefile @@ -15,6 +15,7 @@ dictnames = advancedcpp \ doc_helper \ example01 \ fragile \ + memory_analysis \ operators \ overloads \ pythonizables \ diff --git a/test/cpp/MemoryOwnership/MemOwnrship.apinotes b/test/cpp/MemoryOwnership/MemOwnrship.apinotes new file mode 100644 index 00000000..397d4f94 --- /dev/null +++ b/test/cpp/MemoryOwnership/MemOwnrship.apinotes @@ -0,0 +1,9 @@ +Name: MemOwnrship +Functions: + - Name: memOwnOperatorNew + SwiftReturnOwnership: cppAllocOperatorNew +Tags: + - Name: memOwn + Methods: + - Name: memOwnAllocator + SwiftReturnOwnership: cppAllocNew diff --git a/test/cpp/MemoryOwnership/memory_analysis_redecl.h b/test/cpp/MemoryOwnership/memory_analysis_redecl.h new file mode 100644 index 00000000..d8e9e636 --- /dev/null +++ b/test/cpp/MemoryOwnership/memory_analysis_redecl.h @@ -0,0 +1,10 @@ +#ifndef MEMORY_ANALYSIS_REDECL_H +#define MEMORY_ANALYSIS_REDECL_H +#include "../memory_analysis.h" + +namespace memory { +[[clang::annotate("cppAllocNew")]] +memOwn* allocDefaultMemOwn(); +} + +#endif diff --git a/test/cpp/MemoryOwnership/module.modulemap b/test/cpp/MemoryOwnership/module.modulemap new file mode 100644 index 00000000..123a620b --- /dev/null +++ b/test/cpp/MemoryOwnership/module.modulemap @@ -0,0 +1 @@ +module MemOwnrship { header "../memory_analysis.h" } diff --git a/test/cpp/memory_analysis.cxx b/test/cpp/memory_analysis.cxx new file mode 100644 index 00000000..667515a8 --- /dev/null +++ b/test/cpp/memory_analysis.cxx @@ -0,0 +1,98 @@ +#include "memory_analysis.h" + +namespace memory { + +int memOwn::dtorCount = 0; + +memOwn::memOwn(int value) : val(value) {} + +memOwn::memOwn() { val = 0; } + +memOwn::~memOwn() { ++dtorCount; } + +__attribute__((malloc)) memAnalysisKlass* mallocAttr() { + return (memAnalysisKlass*)malloc(sizeof(memAnalysisKlass)); +} + +__attribute__((ownership_returns(malloc))) memAnalysisKlass* +ownershipReturnsAttr() { + return (memAnalysisKlass*)malloc(sizeof(memAnalysisKlass)); +} + +__attribute__((ownership_takes(malloc, 1))) void +ownershipTakesAttr(memAnalysisKlass* obj) { + free(obj); +} + +void takesPtrButNotOwnership(memAnalysisKlass* obj) { + // Does something with obj but does not take/holds its ownership + return; +} + +// Expected to not return ownership when analysis is off, and there is just +// attr-check +memAnalysisKlass* noAttr() { return new memAnalysisKlass; } + +memOwn* memOwnOperatorNew() { return (memOwn*)::operator new(sizeof(memOwn)); } + +memOwn* allocDefaultMemOwn() { return new memOwn; } + +memOwn* noAttrAlloc() { return new memOwn; } + +memOwn* allocOperatorNewArrAttr(size_t size) { + return (memOwn*)::operator new[](sizeof(memOwn) * size); +} + +memOwn* allocNewArrAttr(int count) { return new memOwn[count]; } + +memOwn* allocMallocAttr(size_t size) { + return (memOwn*)malloc(sizeof(memOwn) * size); +} + +memOwn* allocOperatorNewAttr() { + return (memOwn*)::operator new(sizeof(memOwn)); +} + +memCastDerived* allocMallocDerived() { + return (memCastDerived*)malloc(sizeof(memCastDerived)); +} + +void memOwner::release(memOwn* obj) { delete obj; } + +void memOwner::releaseSecond(memOwn* keep, memOwn* obj) { delete obj; } + +void memOwner::releaseAfterCount(int count, memOwn* obj) { delete obj; } + +void memOwner::keepOwnership(memOwn* obj) { return; } + +void memOwner::releaseStatic(memOwn* obj) { delete obj; } + +void memOwner::releaseOverloaded(memOwn* keep, memOwn* obj) { delete obj; } + +void releaseFree(memOwner* self, memOwn* obj) { delete obj; } + +int* allocIntNewArr(int count) { return new int[count]; } + +int* allocIntOperatorNewArr(size_t count) { + return (int*)::operator new[](count * sizeof(int)); +} + +int* allocIntNew() { return new int; } + +int* allocIntOperatorNew() { return (int*)::operator new(sizeof(int)); } + +int* allocIntMalloc(size_t count) { return (int*)malloc(count * sizeof(int)); } + +int* intNoAttrAlloc() { return new int[5]; } + +void releaseBufferArr(int* buf) { delete[] buf; } + +void releaseBuffer(int* buf) { delete buf; } + +void releaseBufferMalloc(int* buf) { free(buf); } + +void keepBuffer(int* buf) { return; } + +void analyzerNoBodyRelease(memOwn* obj) { delete obj; } + +} // namespace memory diff --git a/test/cpp/memory_analysis.h b/test/cpp/memory_analysis.h new file mode 100644 index 00000000..687a9d57 --- /dev/null +++ b/test/cpp/memory_analysis.h @@ -0,0 +1,163 @@ +#ifndef MEMORY_ANALYSIS_H +#define MEMORY_ANALYSIS_H + +#include +#include +namespace memory { + +class memAnalysisKlass { +public: + int val; +}; +__attribute__((malloc)) memAnalysisKlass* mallocAttr(); +__attribute__((ownership_returns(malloc))) memAnalysisKlass* +ownershipReturnsAttr(); +memAnalysisKlass* noAttr(); + +__attribute__((ownership_takes(malloc, 1))) void +ownershipTakesAttr(memAnalysisKlass* obj); + +void takesPtrButNotOwnership(memAnalysisKlass* obj); + +struct memOwn { + int val; + static int dtorCount; + memOwn(int value); + memOwn(); + // Attribute injected by APINotes + static memOwn* memOwnAllocator(int x) { return new memOwn(x); } + ~memOwn(); +}; + +// Attribute injected by APINotes +memOwn* memOwnOperatorNew(); + +// Attribute injected by redeclaration +memOwn* allocDefaultMemOwn(); + +// No ownership attribute anywhere +memOwn* noAttrAlloc(); + +[[clang::annotate("cppAllocOperatorNewArr")]] +memOwn* allocOperatorNewArrAttr(size_t size); + +[[clang::annotate("cppAllocNewArr")]] +memOwn* allocNewArrAttr(int count); + +[[clang::annotate("cppAllocMalloc")]] +memOwn* allocMallocAttr(size_t size); + +[[clang::annotate("cppAllocOperatorNew")]] +memOwn* allocOperatorNewAttr(); + +// A pair of classes that bind_object can cast between. Neither one is +// polymorphic, so the cast needs no runtime type information. +struct memCastBase { + int base = 0; +}; + +struct memCastDerived : memCastBase { + int derived = 0; +}; + +[[clang::annotate("cppAllocMalloc")]] +memCastDerived* allocMallocDerived(); + +struct memOwner { + + [[clang::annotate("cppDeallocDelete", 2)]] + void release(memOwn* obj); + + [[clang::annotate("cppDeallocDelete", 3)]] + void releaseSecond(memOwn* keep, memOwn* obj); + + [[clang::annotate("cppDeallocDelete", 3)]] + void releaseAfterCount(int count, memOwn* obj); + + [[clang::annotate("cppDeallocNone", 2)]] + void keepOwnership(memOwn* obj); + + [[clang::annotate("cppDeallocDelete", 1)]] + static void releaseStatic(memOwn* obj); + + // One test adds a Python overload to this method, and that overload stays in + // the shared overload group. The other tests use their own methods, so the + // added one reaches none of them. Two parameters make a call with a single + // argument miss this overload, which lets the Python one run. + [[clang::annotate("cppDeallocDelete", 3)]] + void releaseOverloaded(memOwn* keep, memOwn* obj); +}; + +[[clang::annotate("cppDeallocDelete", 2)]] +void releaseFree(memOwner* self, memOwn* obj); + +// A raw pointer to a builtin type binds to a LowLevelView, not an instance +[[clang::annotate("cppAllocNewArr")]] +int* allocIntNewArr(int count); + +[[clang::annotate("cppAllocOperatorNewArr")]] +int* allocIntOperatorNewArr(size_t count); + +[[clang::annotate("cppAllocNew")]] +int* allocIntNew(); + +[[clang::annotate("cppAllocOperatorNew")]] +int* allocIntOperatorNew(); + +[[clang::annotate("cppAllocMalloc")]] +int* allocIntMalloc(size_t count); + +// No ownership attribute anywhere +int* intNoAttrAlloc(); + +[[clang::annotate("cppDeallocDeleteArr", 1)]] +void releaseBufferArr(int* buf); + +[[clang::annotate("cppDeallocDelete", 1)]] +void releaseBuffer(int* buf); + +[[clang::annotate("cppDeallocFree", 1)]] +void releaseBufferMalloc(int* buf); + +[[clang::annotate("cppDeallocNone", 1)]] +void keepBuffer(int* buf); + +// The analyzer reads function bodies, so these must be inline +inline memOwn* analyzerAllocNew() { return new memOwn; } + +inline memOwn* analyzerAllocNewArr() { return new memOwn[5]; } + +inline memOwn* analyzerAllocMalloc() { return (memOwn*)malloc(sizeof(memOwn)); } + +// A method keeps the first answer it gets, so a test that runs with the +// analyzer off needs a function that no other test calls. +inline memOwn* analyzerOffAllocNew() { return new memOwn; } + +inline memOwn* analyzerCachedAllocNew() { return new memOwn; } + +struct analyzerOwner { + void analyzerRelease(memOwn* obj) { delete obj; } + + void analyzerReleaseSecond(memOwn* keep, memOwn* obj) { delete obj; } + + static void analyzerReleaseStatic(memOwn* obj) { delete obj; } + + // the body frees nothing, so the attribute and the analyzer disagree + [[clang::annotate("cppDeallocDelete", 2)]] + void analyzerAttrWins(memOwn* obj) {} + + // the attribute answers for keep, and only the analyzer answers for obj + [[clang::annotate("cppDeallocNone", 2)]] + void analyzerMixedSlots(memOwn* keep, memOwn* obj) { + delete obj; + } + + void analyzerOffRelease(memOwn* obj) { delete obj; } +}; + +inline void analyzerReleaseFree(memOwn* obj) { delete obj; } + +// Defined in memory_analysis.cxx, so the interpreter sees no body of it +void analyzerNoBodyRelease(memOwn* obj); +} // namespace memory +#endif // MEMORY_ANALYSIS_H diff --git a/test/test_memoryanalysis.py b/test/test_memoryanalysis.py new file mode 100644 index 00000000..adb9b312 --- /dev/null +++ b/test/test_memoryanalysis.py @@ -0,0 +1,876 @@ +import gc +import os +import subprocess +import sys + +import py +from pytest import mark +from support import IS_CLING, setup_make + +currpath = py.path.local(__file__).dirpath() +test_dct = str(currpath.join("cpp/memory_analysisDict")) + +FLAGS = "-fmodules -fimplicit-module-maps -fapinotes-modules" +IN_CHILD = "-fapinotes-modules" in os.getenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS", "") + + +def setup_module(mod): + setup_make("memory_analysis") + + +@mark.skipif( + IN_CHILD or IS_CLING, + reason="Cling asserts in collectModuleMaps when built with " + FLAGS, +) +def test00_driver(): + env = os.environ.copy() + env["CPPINTEROP_EXTRA_INTERPRETER_ARGS"] = ( + env.get("CPPINTEROP_EXTRA_INTERPRETER_ARGS", "") + " " + FLAGS + ) + subprocess.check_call([sys.executable, "-m", "pytest", __file__], env=env) + + +class TestMEMORYANALYSIS: + def setup_class(cls): + cls.test_dct = test_dct + import cppjit + + cppjit.add_include_path(str(currpath.join("cpp", "MemoryOwnership"))) + cppjit.include("../memory_analysis.h") + cppjit.include("memory_analysis_redecl.h") + cls.memory_analysis = cppjit.load_library(cls.test_dct + ".so") + + def test01_malloc_attr(self): + import cppjit + + obj = cppjit.gbl.memory.mallocAttr() + assert type(obj) == cppjit.gbl.memory.memAnalysisKlass + assert obj.__python_owns__ + assert obj.__is_malloc__ + + def test02_ownership_returns_attr(self): + import cppjit + + obj = cppjit.gbl.memory.ownershipReturnsAttr() + assert type(obj) == cppjit.gbl.memory.memAnalysisKlass + assert obj.__python_owns__ + assert obj.__is_malloc__ + + def test03_no_attr(self): + import cppjit + + obj = cppjit.gbl.memory.noAttr() + assert type(obj) == cppjit.gbl.memory.memAnalysisKlass + assert not obj.__python_owns__ + obj.__python_owns__ = True + + def test04_redecl_attr(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.allocDefaultMemOwn() + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ + assert not obj.__is_malloc__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 1 + + def test05_redecl_no_attr(self): + import cppjit + + obj = cppjit.gbl.memory.noAttrAlloc() + assert type(obj) == cppjit.gbl.memory.memOwn + assert not obj.__python_owns__ + obj.__python_owns__ = True + + # Setting only python_owns, intends object is allocated with new + assert not obj.__is_malloc__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + + def test08_allocwith_operator_newarr_attr(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.allocOperatorNewArrAttr(5) + assert obj.__python_owns__ + assert obj.__is_no_construct__ + assert obj.__is_array_alloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test09_allocwith_newarr_attr(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.allocNewArrAttr(5) + assert obj.__python_owns__ + assert not obj.__is_no_construct__ + assert obj.__is_array_alloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 5 + + def test10_allocwith_malloc_attr(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.allocMallocAttr(5) + assert obj.__python_owns__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + assert obj.__is_malloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test11_allocwith_operator_new_attr(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.allocOperatorNewAttr() + assert obj.__python_owns__ + assert obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + assert not obj.__is_malloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test12_double_free_with_gblfree(self): + import cppjit + + obj = cppjit.gbl.memory.mallocAttr() + assert obj.__python_owns__ + assert obj.__is_malloc__ + cppjit.gbl.free(obj) + assert not obj.__python_owns__ + + def test13_double_free_with_ownership_takes(self): + import cppjit + + obj = cppjit.gbl.memory.mallocAttr() + assert obj.__python_owns__ + assert obj.__is_malloc__ + cppjit.gbl.memory.ownershipTakesAttr(obj) + assert not obj.__python_owns__ + + def test14_double_free_with_not_ownership_taken(self): + import cppjit + + obj = cppjit.gbl.memory.mallocAttr() + assert obj.__python_owns__ + assert obj.__is_malloc__ + cppjit.gbl.memory.takesPtrButNotOwnership(obj) + assert obj.__python_owns__ + + # The tests below check the mapping from a call argument to the parameter + # that the deallocation attribute labels. Every call releases the object + # in C++, so Python has to give up the ownership of it. + + def owned_memown(self): + import cppjit + + obj = cppjit.gbl.memory.allocDefaultMemOwn() + assert obj.__python_owns__ + return obj + + def test15_release_bound_positional(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.release(obj) + assert not obj.__python_owns__ + assert cppjit.gbl.memory.memOwn.dtorCount == 1 + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 1 + + def test16_release_bound_keyword(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.release(obj=obj) + assert not obj.__python_owns__ + + def test17_release_unbound_positional(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + cppjit.gbl.memory.memOwner.release(owner, obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test18_release_unbound_keyword(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + cppjit.gbl.memory.memOwner.release(owner, obj=obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test19_release_second_positional(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + keep = self.owned_memown() + obj = self.owned_memown() + + owner.releaseSecond(keep, obj) + assert not obj.__python_owns__ + assert keep.__python_owns__ + + def test20_release_second_keyword_reversed(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + keep = self.owned_memown() + obj = self.owned_memown() + + # the values arrive in call order, so only the names say which + # parameter each one belongs to + owner.releaseSecond(obj=obj, keep=keep) + assert not obj.__python_owns__ + assert keep.__python_owns__ + + def test21_release_after_count(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.releaseAfterCount(3, obj) + assert not obj.__python_owns__ + + def test22_release_after_count_keyword(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.releaseAfterCount(count=3, obj=obj) + assert not obj.__python_owns__ + + def test23_keep_ownership(self): + import cppjit + + cppjit.gbl.memory.memOwn.dtorCount = 0 + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.keepOwnership(obj) + assert obj.__python_owns__ + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + owner.keepOwnership(obj=obj) + assert obj.__python_owns__ + + def test24_release_static_through_instance(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + # a static method has no implicit this slot, although the lookup on an + # instance binds self + owner.releaseStatic(obj) + assert not obj.__python_owns__ + + def test25_release_static_through_class(self): + import cppjit + + obj = self.owned_memown() + + cppjit.gbl.memory.memOwner.releaseStatic(obj) + assert not obj.__python_owns__ + + def test26_release_static_with_leading_instance(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + # cppjit drops the leading instance of such a call, so the first + # parameter is the second argument + cppjit.gbl.memory.memOwner.releaseStatic(owner, obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test27_release_free_function(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + cppjit.gbl.memory.releaseFree(owner, obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test28_release_free_function_keyword(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + cppjit.gbl.memory.releaseFree(owner, obj=obj) + assert not obj.__python_owns__ + + def test29_release_free_function_attached(self): + import cppjit + + # a free function whose first parameter is the object works as a + # method, and then cppjit passes self as that parameter + cppjit.gbl.memory.memOwner.releaseAttached = cppjit.gbl.memory.releaseFree + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.releaseAttached(obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test30_release_free_function_attached_keyword(self): + import cppjit + + cppjit.gbl.memory.memOwner.releaseAttached = cppjit.gbl.memory.releaseFree + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + owner.releaseAttached(obj=obj) + assert not obj.__python_owns__ + + # A direct call keeps self inside the argument array, as the vector call + # skips the descriptor. Storing the method first runs the descriptor, and + # then self arrives on its own. + + def test31_release_stored_bound_positional(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + release = owner.release + release(obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test32_release_stored_bound_keyword(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + release = owner.release + release(obj=obj) + assert not obj.__python_owns__ + + def test33_release_second_stored_bound(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + keep = self.owned_memown() + obj = self.owned_memown() + + release_second = owner.releaseSecond + release_second(keep, obj) + assert not obj.__python_owns__ + assert keep.__python_owns__ + + def test34_release_static_stored_bound(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + release_static = owner.releaseStatic + release_static(obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test35_release_free_function_attached_stored_bound(self): + import cppjit + + cppjit.gbl.memory.memOwner.releaseAttached = cppjit.gbl.memory.releaseFree + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + release_attached = owner.releaseAttached + release_attached(obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test36_keep_ownership_stored_bound(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + obj = self.owned_memown() + + keep_ownership = owner.keepOwnership + keep_ownership(obj) + assert obj.__python_owns__ + + def test37_python_overload(self): + import cppjit + + owner = cppjit.gbl.memory.memOwner() + keep = self.owned_memown() + obj = self.owned_memown() + + def extra(self, obj): + return 0 + + cppjit.gbl.memory.memOwner.releaseOverloaded.__add_overload__(extra) + + # a Python overload carries no ownership information, so the arguments + # of such a call keep their ownership + owner.releaseOverloaded(obj) + assert obj.__python_owns__ + assert owner.__python_owns__ + + # the name of a Python argument is no C++ parameter name either + owner.releaseOverloaded(obj=obj) + assert obj.__python_owns__ + + # the C++ overload needs two arguments, and it does take the ownership + owner.releaseOverloaded(keep, obj) + assert not obj.__python_owns__ + assert keep.__python_owns__ + + def test38_bind_object_keeps_the_allocation_kind(self): + import cppjit + + obj = cppjit.gbl.memory.allocMallocDerived() + assert obj.__python_owns__ + assert obj.__is_malloc__ + + # a cast moves the ownership to the new proxy, so the deallocator of + # the memory has to move with it + cast = cppjit.bind_object(obj, cppjit.gbl.memory.memCastBase) + assert not obj.__python_owns__ + assert cast.__python_owns__ + assert cast.__is_malloc__ + + del cast + gc.collect() + + +class TestLOWLEVELVIEWMEMORY: + def setup_class(cls): + cls.test_dct = test_dct + import cppjit + + cppjit.add_include_path(str(currpath.join("cpp", "MemoryOwnership"))) + cppjit.include("../memory_analysis.h") + cls.memory_analysis = cppjit.load_library(cls.test_dct + ".so") + + def test01_newarr_attr(self): + import cppjit + import cppjit.types + + view = cppjit.gbl.memory.allocIntNewArr(4) + assert type(view) == cppjit.types.LowLevelView + assert view.__python_owns__ + assert view.__cpp_array__ + assert not view.__is_new__ + + # the view frees the buffer with ::operator delete[] + del view + gc.collect() + + def test02_operator_newarr_attr(self): + import cppjit + + view = cppjit.gbl.memory.allocIntOperatorNewArr(4) + assert view.__python_owns__ + assert view.__cpp_array__ + assert not view.__is_new__ + + del view + gc.collect() + + def test03_new_attr(self): + import cppjit + + view = cppjit.gbl.memory.allocIntNew() + assert view.__python_owns__ + assert view.__is_new__ + assert not view.__cpp_array__ + + # the view frees the buffer with ::operator delete + del view + gc.collect() + + def test04_operator_new_attr(self): + import cppjit + + view = cppjit.gbl.memory.allocIntOperatorNew() + assert view.__python_owns__ + assert view.__is_new__ + assert not view.__cpp_array__ + + del view + gc.collect() + + def test05_malloc_attr(self): + import cppjit + + view = cppjit.gbl.memory.allocIntMalloc(4) + assert view.__python_owns__ + # malloc has no flag of its own: no allocation flag means free() + assert not view.__cpp_array__ + assert not view.__is_new__ + + del view + gc.collect() + + def test06_no_attr(self): + import cppjit + + view = cppjit.gbl.memory.intNoAttrAlloc() + assert not view.__python_owns__ + + # function has no attribute, so the flags stay as ll_new wrote them: + # the user takes ownership, therefore the default delete[] is called + view.__python_owns__ = True + assert view.__cpp_array__ + assert not view.__is_new__ + + del view + gc.collect() + + def test07_release_newarr_buffer(self): + import cppjit + + view = cppjit.gbl.memory.allocIntNewArr(4) + assert view.__python_owns__ + + cppjit.gbl.memory.releaseBufferArr(view) + assert not view.__python_owns__ + + del view + gc.collect() + + def test08_release_newarr_buffer_keyword(self): + import cppjit + + view = cppjit.gbl.memory.allocIntNewArr(4) + + cppjit.gbl.memory.releaseBufferArr(buf=view) + assert not view.__python_owns__ + + def test09_release_new_buffer(self): + import cppjit + + view = cppjit.gbl.memory.allocIntNew() + + cppjit.gbl.memory.releaseBuffer(view) + assert not view.__python_owns__ + + del view + gc.collect() + + def test10_release_malloc_buffer(self): + import cppjit + + view = cppjit.gbl.memory.allocIntMalloc(4) + + cppjit.gbl.memory.releaseBufferMalloc(view) + assert not view.__python_owns__ + + del view + gc.collect() + + def test11_keep_buffer(self): + import cppjit + + view = cppjit.gbl.memory.allocIntNewArr(4) + + # a cppDeallocNone parameter leaves the ownership where it was + cppjit.gbl.memory.keepBuffer(view) + assert view.__python_owns__ + + del view + gc.collect() + + def test12_release_with_free(self): + import cppjit + + view = cppjit.gbl.malloc(16) + + cppjit.gbl.free(view) + assert not view.__python_owns__ + + del view + gc.collect() + + +class TestMEMORYANALYZER: + def setup_class(cls): + cls.test_dct = test_dct + import cppjit + + cppjit.add_include_path(str(currpath.join("cpp", "MemoryOwnership"))) + cppjit.include("../memory_analysis.h") + cls.memory_analysis = cppjit.load_library(cls.test_dct + ".so") + + def teardown_class(cls): + import cppjit + + # the flag is global, so the tests that follow must not inherit it + cppjit.use_memory_analyzer(False) + + def test01_alloc_with_analyzer_off(self): + import cppjit + + cppjit.use_memory_analyzer(False) + + obj = cppjit.gbl.memory.analyzerOffAllocNew() + assert not obj.__python_owns__ + + obj.__python_owns__ = True + + def test02_alloc_with_new(self): + import cppjit + + cppjit.use_memory_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.analyzerAllocNew() + assert obj.__python_owns__ + assert not obj.__is_array_alloc__ + assert not obj.__is_malloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 1 + + def test03_alloc_with_newarr(self): + import cppjit + + cppjit.use_memory_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.analyzerAllocNewArr() + assert obj.__python_owns__ + assert obj.__is_array_alloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 5 + + def test04_alloc_with_malloc(self): + import cppjit + + cppjit.use_memory_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + obj = cppjit.gbl.memory.analyzerAllocMalloc() + assert obj.__python_owns__ + assert obj.__is_malloc__ + + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test05_cached_answer_(self): + import cppjit + + cppjit.use_memory_analyzer(True) + + first = cppjit.gbl.memory.analyzerCachedAllocNew() + assert first.__python_owns__ + + cppjit.use_memory_analyzer(False) + second = cppjit.gbl.memory.analyzerCachedAllocNew() + assert second.__python_owns__ + + def test06_release_bound_positional(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + obj = cppjit.gbl.memory.memOwn(5) + assert obj.__python_owns__ + + owner.analyzerRelease(obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test07_release_bound_keyword(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerRelease(obj=obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test08_release_unbound_positional(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + obj = cppjit.gbl.memory.memOwn(5) + + cppjit.gbl.memory.analyzerOwner.analyzerRelease(owner, obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test09_release_second_parameter(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + keep = cppjit.gbl.memory.memOwn(5) + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerReleaseSecond(keep, obj) + assert not obj.__python_owns__ + assert keep.__python_owns__ + assert owner.__python_owns__ + + def test10_release_second_parameter_keyword_reversed(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + keep = cppjit.gbl.memory.memOwn(5) + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerReleaseSecond(obj=obj, keep=keep) + assert not obj.__python_owns__ + assert keep.__python_owns__ + + def test11_release_static_from_class(self): + import cppjit + + cppjit.use_memory_analyzer(True) + obj = cppjit.gbl.memory.memOwn(5) + + cppjit.gbl.memory.analyzerOwner.analyzerReleaseStatic(obj) + assert not obj.__python_owns__ + + def test12_release_static_from_instance(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerReleaseStatic(obj) + assert not obj.__python_owns__ + assert owner.__python_owns__ + + def test13_release_free_function(self): + import cppjit + + cppjit.use_memory_analyzer(True) + obj = cppjit.gbl.memory.memOwn(5) + + cppjit.gbl.memory.analyzerReleaseFree(obj) + assert not obj.__python_owns__ + + def test14_attribute_wins(self): + import cppjit + + cppjit.use_memory_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + owner = cppjit.gbl.memory.analyzerOwner() + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerAttrWins(obj) + # the attribute says delete although the body frees nothing, and the + # attribute wins + assert not obj.__python_owns__ + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + obj.__python_owns__ = True + del obj + gc.collect() + assert cppjit.gbl.memory.memOwn.dtorCount == 1 + + def test15_attribute_and_body_on_different_arg(self): + import cppjit + + cppjit.use_memory_analyzer(True) + owner = cppjit.gbl.memory.analyzerOwner() + keep = cppjit.gbl.memory.memOwn(5) + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerMixedSlots(keep, obj) + assert keep.__python_owns__ + assert not obj.__python_owns__ + + def test16_release_with_analyzer_off(self): + import cppjit + + cppjit.use_memory_analyzer(False) + owner = cppjit.gbl.memory.analyzerOwner() + obj = cppjit.gbl.memory.memOwn(5) + + owner.analyzerOffRelease(obj) + assert obj.__python_owns__ + + obj.__python_owns__ = False + + def test17_release_without_a_body(self): + import cppjit + + cppjit.use_memory_analyzer(True) + obj = cppjit.gbl.memory.memOwn(5) + + cppjit.gbl.memory.analyzerNoBodyRelease(obj) + assert obj.__python_owns__ + + obj.__python_owns__ = False + + +@mark.skipif(not IN_CHILD, reason="needs " + FLAGS) +class TestMEMORYANALYSIS_APINOTES: + def setup_class(cls): + cls.test_dct = test_dct + import cppjit + + cppjit.add_include_path(str(currpath.join("cpp", "MemoryOwnership"))) + cppjit.include("../memory_analysis.h") + cls.memory_analysis = cppjit.load_library(cls.test_dct + ".so") + + def test01_apinotes_attr_method(self): + import cppjit + + obj = cppjit.gbl.memory.memOwn.memOwnAllocator(5) + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ + + def test02_apinotes_attr_func(self): + import cppjit + + obj = cppjit.gbl.memory.memOwnOperatorNew() + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ + assert obj.__is_no_construct__