Use SWIG differently, so we have a single Python binary instead of one per Python version.

docs/source/Installation/installation.rst

@@ -5,7 +5,7 @@ Installing RoadRunner
 Python Front End
 ===================
 
-The recommended way to use roadrunner is by installing the Python front end and writing Python scripts.
+The recommended way to use roadrunner is by installing the Python front end and writing Python scripts, for Python 3.10 or later:
 
 .. code-block:: bash
 
@@ -158,15 +158,14 @@ These are controlled by turning on or off desired options in cmake.
 Build the Python Bindings
 -------------------------
 
-    1) Download `SWIG version 3.0.0 <https://sourceforge.net/projects/swig/files/>`_ for linux, macos or windows. It is just a zip file - decompress it and put it where you want it. SWIG is now installed and the swig executable is under the top level swig directory.
+    1) Download `SWIG version 4.4.1 <https://sourceforge.net/projects/swig/files/>`_ (or later) for linux, macos or windows. It is just a zip file - decompress it and put it where you want it. SWIG is now installed and the swig executable is under the top level swig directory.
 
     2) Configure or reconfigure cmake using the `-DBUILD_PYTHON=ON` option. If you installed swig
     globally or add the swig directory to the PATH environment variable, you will *not* need
     the `-DSWIG_EXECUTABLE` argument as `swig.exe` will be found
     automatically. However in most circumstances you will need to provide the path to the `swig.exe` using
     `-DSWIG_EXECUTABLE`. As per the instructions for [FindPython](https://cmake.org/cmake/help/git-stage/module/FindPython.html)
-    you can build for specific Python environments by pointing -DPython_ROOT_DIR to the root of your Python installation. Using Conda
-    environments and this variable makes it easy to build Python wheels for different Python versions.
+    you can build for specific Python environments by pointing -DPython_ROOT_DIR to the root of your Python installation.
 
 .. code-block:: bash
 
@@ -176,7 +175,7 @@ Build the Python Bindings
         -DCMAKE_BUILD_TYPE="Release" \
         -DBUILD_PYTHON=ON \
         -DPython_ROOT_DIR="/path/to/miniconda/env/root"
-        -DSWIG_EXECUTABLE="path/to/swig3.0.0/executable/swig.exe" ..
+        -DSWIG_EXECUTABLE="path/to/swig4.x/executable/swig.exe" ..
     cmake --build . --target install --config Release
 
 

requirements.txt

@@ -1 +1 @@
-numpy~=2.2
+numpy>=1.21.3

rrplugins/wrappers/python/setup_rrplugins.py

@@ -52,8 +52,10 @@ def has_ext_modules(foo):
 #        'rrplugins.testing' : ['*.xml', '*.txt', '*.dat', 'dsmts/*.xml', 'dsmts/*.csv', 'test_data/*']
     },
     install_requires=[
-          'numpy>=2.4',
+          'numpy>=1.21.3',
           'matplotlib>=3',
           ],
     distclass=BinaryDistribution,
+    # No compiled Python extension is involved, so one wheel serves every Python from 3.10 on.
+    options={'bdist_wheel': {'py_limited_api': 'cp310'}},
 )

rrplugins/wrappers/python/telplugins.py

@@ -156,10 +156,10 @@ def fromNumPy(cls, numPyData):
             raise ValueError ('fromNumPy only accepts two dimensional arrays')
 
         # If there are no column names then make some up
-        if hasattr(numPyData, "colnames") and not numPyData.colnames == None:
-            colHdr  = numPyData.colnames
-        elif not numPyData.colnames == None:
-            colHdr  = numPyData.colnames
+        if getattr(numPyData, "colnames", None):
+            colHdr  = list(numPyData.colnames)
+        elif numPyData.dtype.names is not None:
+            colHdr  = list(numPyData.dtype.names)
         else:
             colHdr = []
             for i in range (nrCols):

source/RegistrationFactory.cpp

@@ -39,6 +39,14 @@ namespace rr {
     void RegistrationFactory::registerSolver(Registrable* i) {
         if (!i)
             throw CoreException("Registrar is null");
+        // Several copies of this code in one process (roadrunner and rrplugins, say) can share
+        // one factory, so a solver that is already registered is not added a second time.
+        for (auto existing : registrars) {
+            if (existing->getName() == i->getName()) {
+                delete i;
+                return;
+            }
+        }
         registrars.push_back(i);
     }
 

test/CMakeLists.txt

@@ -122,8 +122,8 @@ endfunction()
 #   to be added to the swig target.
 macro(add_rr_test_swig_library SWIG_TARGET_NAME SWIG_INTERFACE_FILE)
     # no harm in finding python here
-    find_package(Python COMPONENTS Interpreter Development NumPy REQUIRED)
-    find_package(SWIG 4.0.1 REQUIRED
+    find_package(Python COMPONENTS Interpreter Development REQUIRED)
+    find_package(SWIG 4.4.1 REQUIRED
             COMPONENTS python
             )
     include(UseSWIG)
@@ -176,15 +176,21 @@ macro(add_rr_test_swig_library SWIG_TARGET_NAME SWIG_INTERFACE_FILE)
             $<TARGET_PROPERTY:roadrunner,INCLUDE_DIRECTORIES>
             "${RR_DEPENDENCIES_INSTALL_PREFIX}/include/rr-libstruct"
             "${Python_INCLUDE_DIRS}"
-            "${Python_NumPy_INCLUDE_DIRS}"
             $<TARGET_PROPERTY:roadrunner_python,INCLUDE_DIRECTORIES>
             )
+
+    # Like the roadrunner module, the test modules only use the Python stable ABI, so the
+    # installed wheel can load them from any Python from 3.10 on. Match roadrunner/CMakeLists.txt.
+    target_compile_definitions(${SWIG_TARGET_NAME} PRIVATE Py_LIMITED_API=0x030A0000)
+
     if (NOT APPLE)
+        if (WIN32)
+            # Py_LIMITED_API makes pyconfig.h link python3.lib, so only the search path is needed.
+            target_link_directories(${SWIG_TARGET_NAME} PRIVATE ${Python_LIBRARY_DIRS})
+        endif ()
         target_link_libraries(${SWIG_TARGET_NAME} PUBLIC
                 rr-mockups
                 roadrunner-static
-                Python::Module # links the Python library only where a module must (Windows); includes come with it
-                Python::NumPy  # remember, includes will just come with linking Python::NumPy
                 )
     else ()
 
@@ -338,7 +344,9 @@ if (BUILD_PYTHON AND BUILD_TESTS)
     add_pytest_to_ctest(python_tests_SteadyStateSolverTests "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_steadyStateSolver.py")
     add_pytest_to_ctest(python_tests_StructuralAnalysisTests "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_structuralAnalysis.py")
     add_pytest_to_ctest(python_tests_RoadRunnerMap "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_roadrunner_map.py")
-    add_pytest_to_ctest(python_tests_AutoPluginTest "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_auto_plugin.py")
+    if (BUILD_RR_PLUGINS)
+        add_pytest_to_ctest(python_tests_AutoPluginTest "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_auto_plugin.py")
+    endif ()
     #add_pytest_to_ctest(python_tests_PickleTests "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_pickle.py")
     add_pytest_to_ctest(python_tests_RRTests "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_rrtests.py")
     add_pytest_to_ctest(python_tests_SettingTests "${RR_PYTHON_TESTING_BUILD_PREFIX}/test_settings.py")

test/TestModelFactory.i

@@ -1,8 +1,13 @@
 %module TestModelFactory
 
+%begin %{
+#if defined(_WIN32) && !defined(NOMINMAX)
+#define NOMINMAX
+#endif
+%}
+
 %{
     #define SWIG_FILE_WITH_INIT
-    #define PY_ARRAY_UNIQUE_SYMBOL RoadRunner_ARRAY_API // for the NamedArrayObject (see roadrunner.i)
 
     #include "PyUtils.h" // for variant_to_py and back.
     #include "Matrix.h"
@@ -15,11 +20,6 @@
 
 //// since we be using the ls::Matrix from a Python
 //// we'll be using the same API needed by the main roadrunner bindings
-%include "numpy.i"
-%init %{
-import_array();
-rr::pyutil_init(m);
-%}
 //
 #include <vector>
 %include <typemaps.i>
@@ -158,24 +158,9 @@ rr::pyutil_init(m);
  */
 %typemap(out) std::vector<double> {
     // marker for a std::vector<double> typemap
-    const npy_intp length = $1.size();
-
-    // create a numpy type
-    PyArray_Descr *desc = PyArray_DescrFromType(NPY_DOUBLE);
-
-    // copy the data from the std::vector<double> so that we can give the
-    // ptr to the numpy array
-    // hopefully this will be cleaned up by numpy...
-    double* data = new double[length];
-    std::copy($1.begin(), $1.end(), data);
-    $result = PyArray_SimpleNewFromData(1, &length, NPY_DOUBLE, (void*)data);
-
-    PyArrayObject * arr = reinterpret_cast<PyArrayObject*>($result);
-
-    PyArray_ENABLEFLAGS(arr, NPY_ARRAY_OWNDATA);
-    int owned = PyArray_CHKFLAGS(arr, NPY_ARRAY_OWNDATA);
-    if (!owned){
-        std::cerr << "PyArrayObject does not own its memory" << std::endl;
+    $result = rr::doubles_to_py($1.data(), $1.size());
+    if (!$result) {
+        SWIG_fail;
     }
 }
 
@@ -191,14 +176,26 @@ rr::pyutil_init(m);
     PyObject* colnames = matrix3DtoNumpy.convertColNames();
     PyObject* rownames = matrix3DtoNumpy.convertRowNames();
 
+    if (!idx || !npArray3D || !rownames || !colnames) {
+        Py_XDECREF(idx);
+        Py_XDECREF(npArray3D);
+        Py_XDECREF(rownames);
+        Py_XDECREF(colnames);
+        SWIG_fail;
+    }
+
     $result = PyTuple_Pack(4, idx, npArray3D, rownames, colnames);
+    Py_DECREF(idx);
+    Py_DECREF(npArray3D);
+    Py_DECREF(rownames);
+    Py_DECREF(colnames);
 }
 
 // Typemap for a C++ string
 %typemap(in) str {
     // typemap for Python -> C string
   if (PyUnicode_Check($input)) {
-    $1 = std::string(PyUnicode_AsUTF8($input));
+    $1 = std::string(PyUnicode_AsUTF8AndSize($input, nullptr));
   } else {
     SWIG_exception(SWIG_TypeError, "string expected");
   }
@@ -290,39 +287,18 @@ rr::pyutil_init(m);
 
     if (iscpx) {
         // if needed, we use complex
-        size_t len = $1.size();
-        npy_intp dims[1] = {static_cast<npy_intp>(len)};
-
-        PyObject *array = PyArray_SimpleNew(1, dims, NPY_COMPLEX128);
-
-        if (!array) {
-            // TODO error handling.
-            return 0;
-        }
-
-        cpx *data = (cpx*)PyArray_DATA((PyArrayObject*)array);
-
-        memcpy(data, &vec[0], sizeof(std::complex<double>)*len);
-
-        $result  = array;
+        $result = rr::complexes_to_py(vec.data(), vec.size());
     } else {
         // otherwise we just use double
-        size_t len = $1.size();
-        npy_intp dims[1] = {static_cast<npy_intp>(len)};
-
-        PyObject *array = PyArray_SimpleNew(1, dims, NPY_DOUBLE);
-
-        if (!array) {
-            // TODO error handling.
-            return 0;
-        }
-
-        double *data = (double*)PyArray_DATA((PyArrayObject*)array);
-        for (int i = 0; i < vec.size(); ++i) {
-            data[i] = std::real(vec[i]);
+        std::vector<double> reals;
+        reals.reserve(vec.size());
+        for (const cpx& value : vec) {
+            reals.push_back(std::real(value));
         }
-
-        $result  = array;
+        $result = rr::doubles_to_py(reals.data(), reals.size());
+    }
+    if (!$result) {
+        SWIG_fail;
     }
 }
 

test/cxx_api_tests/SettingTestsSwigAPI.i

@@ -1,16 +1,19 @@
 %module SettingTestsSwigAPI
 
+%begin %{
+#if defined(_WIN32) && !defined(NOMINMAX)
+#define NOMINMAX
+#endif
+%}
+
 %{
 #include <unordered_map>
 #include "Setting.h"
 #include <variant> // for visit
-#define PY_ARRAY_UNIQUE_SYMBOL RoadRunner_ARRAY_API
-#include <numpy/arrayobject.h>
 #include "PyUtils.h"
 #include "SettingTestsSwigAPI.h"
 %};
 #include "PyUtils.h"
-#include <numpy/arrayobject.h>
 
 %include "exception.i"
 

test/python/test_namedArray.py

@@ -330,5 +330,113 @@ def test_viewcast_masked_array_to_NamedArray(self):
         n = m.view(NamedArray)
         self.assertIsInstance(n, NamedArray)
 
+
+class NamedArrayCompatibilityTests(unittest.TestCase):
+    """Behavior of NamedArray as seen through the compiled module."""
+
+    def simulation(self):
+        from roadrunner import RoadRunner
+        from roadrunner.tests import TestModelFactory as tmf
+        m = RoadRunner(tmf.SimpleFlux().str())
+        return m, m.simulate(0, 10, 11)
+
+    def test_class_is_reachable_from_old_and_new_locations(self):
+        import roadrunner.named_array
+        import roadrunner._roadrunner
+        self.assertIs(roadrunner._roadrunner.NamedArray, roadrunner.named_array.NamedArray)
+        self.assertEqual(NamedArray.__module__, "roadrunner._roadrunner")
+        self.assertTrue(issubclass(NamedArray, np.ndarray))
+
+    def test_simulation_result_type_and_dtype(self):
+        m, sim = self.simulation()
+        self.assertIsInstance(sim, NamedArray)
+        self.assertEqual(sim.dtype, np.float64)
+        self.assertEqual(sim.rownames, [])
+        self.assertEqual(sim.colnames, ['time', '[S1]', '[S2]'])
+
+    def test_simulation_result_index_by_column_name(self):
+        m, sim = self.simulation()
+        col = sim['time']
+        self.assertEqual(col.shape, (11,))
+        np.testing.assert_array_equal(col, sim[:, 0])
+        np.testing.assert_allclose(col, np.linspace(0, 10, 11))
+        np.testing.assert_array_equal(sim['[S2]'], sim[:, 2])
+
+    def test_simulation_result_str_shows_names_and_every_row(self):
+        m, sim = self.simulation()
+        text = str(sim)
+        for name in sim.colnames:
+            self.assertIn(name, text)
+        self.assertGreaterEqual(len(text.splitlines()), 11)
+        self.assertEqual(text, repr(sim))
+
+    def test_simulation_result_is_independent_of_later_simulations(self):
+        m, sim = self.simulation()
+        expected = np.array(sim)
+        m.reset()
+        m.simulate(0, 20, 21)
+        self.assertEqual(sim.shape, (11, 3))
+        np.testing.assert_array_equal(sim, expected)
+
+    def test_simulation_result_is_writeable_without_changing_the_model(self):
+        m, sim = self.simulation()
+        before = m.model.getFloatingSpeciesConcentrations().copy()
+        sim[:, 1:] = 12345.0
+        np.testing.assert_array_equal(m.model.getFloatingSpeciesConcentrations(), before)
+
+    def test_pickle_round_trip_of_simulation_result(self):
+        m, sim = self.simulation()
+        loaded = pickle.loads(pickle.dumps(sim))
+        self.assertIsInstance(loaded, NamedArray)
+        self.assertEqual(loaded.colnames, sim.colnames)
+        self.assertEqual(loaded.rownames, sim.rownames)
+        np.testing.assert_array_equal(loaded, sim)
+
+    def test_pickle_refers_to_the_old_module_path(self):
+        n = np.zeros((2, 3)).view(NamedArray)
+        self.assertIn(b"roadrunner._roadrunner", pickle.dumps(n))
+
+    def test_pickle_written_in_the_original_format_still_loads(self):
+        values = np.arange(6, dtype=np.float64)
+
+        class OriginalPickle:
+            def __reduce__(self):
+                state = {
+                    'array': values.tobytes(),
+                    'nDims': 2,
+                    'dim1': 2,
+                    'dim2': 3,
+                    'rownames': ['R1', 'R2'],
+                    'colnames': ['C1', 'C2', 'C3'],
+                    '_pickle_version': 5,
+                }
+                return NamedArray, ((2, 3),), state
+
+        loaded = pickle.loads(pickle.dumps(OriginalPickle()))
+        self.assertIsInstance(loaded, NamedArray)
+        np.testing.assert_array_equal(loaded, values.reshape(2, 3))
+        self.assertEqual(loaded.rownames, ['R1', 'R2'])
+        self.assertEqual(loaded.colnames, ['C1', 'C2', 'C3'])
+
+    def test_jacobian_has_matching_row_and_column_names(self):
+        from roadrunner import RoadRunner
+        from roadrunner.tests import TestModelFactory as tmf
+        m = RoadRunner(tmf.SimpleFlux().str())
+        j = m.getFullJacobian()
+        self.assertIsInstance(j, NamedArray)
+        self.assertEqual(j.shape[0], j.shape[1])
+        self.assertEqual(len(j.rownames), j.shape[0])
+        self.assertEqual(j.rownames, j.colnames)
+        np.testing.assert_array_equal(j[j.colnames[0]], j[:, 0])
+
+    def test_slices_and_ufuncs_stay_named_arrays(self):
+        m, sim = self.simulation()
+        self.assertIsInstance(sim[2:5, :], NamedArray)
+        self.assertIsInstance(sim * 2, NamedArray)
+        self.assertEqual((sim * 2).shape, sim.shape)
+        self.assertIsInstance(np.asarray(sim), np.ndarray)
+        self.assertNotIsInstance(np.asarray(sim), NamedArray)
+
+
 if __name__ == "__main__":
     unittest.main()