Optionally store StaticPyObjStore on micropython heap via root pointer

.appveyor.yml

@@ -5,17 +5,30 @@ environment:
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
       MICROPY_CPYTHON3: c:/python39/python.exe
       FullTypeCheck: 0
+      StaticPyObjRootPtr: 0
+      BuildMsys2Version: 1
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+      MICROPY_CPYTHON3: c:/python39/python.exe
+      FullTypeCheck: 0
+      StaticPyObjRootPtr: 1
       BuildMsys2Version: 1
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       MICROPY_CPYTHON3: c:/python38/python.exe
       FullTypeCheck: 1
+      StaticPyObjRootPtr: 0
       BuildMsys2Version: 0
     - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu
       MICROPY_CPYTHON3: python3
       FullTypeCheck: 0
+      StaticPyObjRootPtr: 0
+    - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu
+      MICROPY_CPYTHON3: python3
+      FullTypeCheck: 1
+      StaticPyObjRootPtr: 0
     - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu
       MICROPY_CPYTHON3: python3
       FullTypeCheck: 1
+      StaticPyObjRootPtr: 1
 
 configuration:
 - Debug
@@ -99,7 +112,7 @@ for:
         C:\msys64\usr\bin\bash.exe -l -c "make V=1 MICROPYTHON_PORT_DIR=../micropython/ports/windows clean"
         # The redirection is because 'make submodules' might get called, which in turn calls git commands which write to stderr
         # and that is considered an error on Appveyor. And GIT_REDIRECT_STDERR seems to have no effect on the git version used in this msys2.
-        &{C:\msys64\usr\bin\bash.exe -l -c "make V=1 PYTHON=/usr/bin/python3 MICROPYTHON_PORT_DIR=../micropython/ports/windows test"} 2>&1
+        &{C:\msys64\usr\bin\bash.exe -l -c "make V=1 CPPFLAGS_EXTRA='-DUPYWRAP_FULLTYPECHECK=$env:FullTypeCheck -DUPYWRAP_STATICPYOBJ_USE_ROOTPTR=$env:StaticPyObjRootPtr' PYTHON=/usr/bin/python3 MICROPYTHON_PORT_DIR=../micropython/ports/windows test"} 2>&1
         if ($LASTEXITCODE -ne 0) {
           throw "MSYS2 build exited with code $LASTEXITCODE"
         }
@@ -112,7 +125,7 @@ for:
 
   build_script:
   - ps: |
-      make test CC=gcc-9 CXX=g++-9 CPPFLAGS_EXTRA="-DUPYWRAP_FULLTYPECHECK=$env:FullTypeCheck"
+      make test CC=gcc-9 CXX=g++-9 CPPFLAGS_EXTRA="-DUPYWRAP_FULLTYPECHECK=$env:FullTypeCheck -DUPYWRAP_STATICPYOBJ_USE_ROOTPTR=$env:StaticPyObjRootPtr"
 
 on_failure:
 - ps: |

Makefile

@@ -46,7 +46,7 @@ endif
 V ?= 0
 MAKEOPTS ?= -j4 V=$(V) VARIANT=$(VARIANT) VARIANT_DIR=$(VARIANT_DIR)
 MAKEUPY = make -C $(MICROPYTHON_PORT_DIR) $(MAKEOPTS)
-UPYFLAGS = MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_USSL=0 MICROPY_PY_AXTLS=0 MICROPY_FATFS=0 MICROPY_PY_THREAD=0
+UPYFLAGS = QSTR_GLOBAL_DEPENDENCIES=$(CURDIR)/mpy_wrap_root_pointer.c MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_USSL=0 MICROPY_PY_AXTLS=0 MICROPY_FATFS=0 MICROPY_PY_THREAD=0
 
 # Extra features we need; note that for the ports tested so far this is already enabled or else
 # defined conditionally so we can just define it here again. Otherwise make this conditional

README.md

@@ -71,6 +71,21 @@ import foo
 print(foo.TransformList(['a', 'b']))  # Prints ['aTRANSFORM', 'bTRANSFORM']
 ```
 
+When using the `UPYWRAP_STATICPYOBJ_USE_ROOTPTR` configuration option (opt-in):
+
+The micropython-wrap type table needs to be registered as root pointer. Safe this as a `.c` file and add it to micropythons
+`MICROPY_SOURCE_QSTR` sources in cmake or `SRC_QSTR` in make. This is to make sure it is added to the global states root pointers
+and thus not collected by micropythons garbage-collector.
+
+```c
+#include "py/obj.h"
+
+//micropythons global storage needs to be registered as root pointer to avoid garbage collection
+//the user needs to do this at least once to be able to compile micropython-wrap
+MP_REGISTER_ROOT_POINTER(mp_map_t* micropython_wrap_global_storage);
+```
+
+
 Type Conversion
 ---------------
 Conversion between standard native types and `mp_obj_t`, the MicroPython opaque object type

detail/configuration.h

@@ -127,4 +127,15 @@
 #define UPYWRAP_MAXNUMKWARGS (8)
 #endif
 
+//Store the StaticPyObjectStore in a root pointer map instead of a random module's globals dict.
+//Enabling this requires the user to add the micropython root pointer in one micropython source file:
+//
+//  MP_REGISTER_ROOT_POINTER(mp_map_t* micropython_wrap_global_storage);
+//
+//The benefit of this is that the StaticPyObjectStore is now hidden from the mpy runtime.
+//This is disabled by default for backwards compatibility.
+#ifndef UPYWRAP_STATICPYOBJ_USE_ROOTPTR
+#define UPYWRAP_STATICPYOBJ_USE_ROOTPTR (0)
+#endif
+
 #endif

detail/micropython.h

@@ -1,6 +1,7 @@
 #ifndef MICROPYTHON_WRAP_DETAIL_MICROPYTHON_H
 #define MICROPYTHON_WRAP_DETAIL_MICROPYTHON_H
 
+#include "mpmap.h"
 #include "micropythonc.h"
 #include <cmath>
 #include <cstdint>
@@ -11,6 +12,32 @@
 
 namespace upywrap
 {
+#if UPYWRAP_STATICPYOBJ_USE_ROOTPTR
+  /**
+  * @brief This is the global map in the micropython state which is used to store the class types and function pointers.
+  *
+  * All objects that micropython-wrap allocates are referenced by this map, or by maps in this map.
+  * It needs to be initialized only once, but can be reset by assigning a new empty map to it.
+  * This reset function can be used to reset the micropython-wrap state stored on the micropython heap.
+  * It MUST be called, if the micropython heap is reset externally.
+  */
+  inline void reset_mpy_wrap_global_map() {
+    MP_STATE_VM(micropython_wrap_global_storage) = m_new0(mp_map_t, 1);
+  }
+
+  inline mp_map_t* get_mpy_wrap_global_map() {
+    return MP_STATE_VM(micropython_wrap_global_storage);
+  }
+
+  inline void init_mpy_wrap_global_map() {
+    static bool init = false;
+    if (!init) {
+      init = true;
+      reset_mpy_wrap_global_map();
+    }
+  }
+#endif
+
   inline mp_obj_t new_qstr( qstr what )
   {
     return MP_OBJ_NEW_QSTR( what );
@@ -367,6 +394,12 @@ namespace upywrap
       return *List();
     }
 
+    static void ResetBackEnd()
+    {
+      // Other functions like InitBackEnd will check if it's already initialized
+      *List() = nullptr;
+    }
+
     static bool Initialized()
     {
       return !!*List();
@@ -481,9 +514,23 @@ namespace upywrap
       */
   inline void InitializePyObjectStore( mp_obj_dict_t& dict )
   {
-    if( !StaticPyObjectStore::Initialized() )
+#if UPYWRAP_STATICPYOBJ_USE_ROOTPTR
+    (void) dict;
+    upywrap::init_mpy_wrap_global_map();
+    MPyMapView<qstr, mp_obj_list_t*> mpy_wrap_map { get_mpy_wrap_global_map() };
+#else
+    if( StaticPyObjectStore::Initialized() ) {
+      return;
+    }
+    MPyMapView<qstr, mp_obj_list_t*> mpy_wrap_map { &dict.map };
+#endif
+
+    qstr static_py_obj_store_key = qstr_from_str( "_StaticPyObjectStore" );
+    if( !mpy_wrap_map.contains( static_py_obj_store_key ) )
     {
-      mp_obj_dict_store( &dict, new_qstr( "_StaticPyObjectStore" ), StaticPyObjectStore::InitBackEnd() );
+      // If we can't find it in the map, the mpy state is either fresh or was reset. Reinit from scratch.
+      StaticPyObjectStore::ResetBackEnd();
+      mpy_wrap_map[static_py_obj_store_key] = StaticPyObjectStore::InitBackEnd();
     }
   }
 

detail/mpmap.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "micropythonc.h"
+#include <type_traits>
+
+namespace upywrap {
+/**
+ * @brief A wrapper around the MicroPython map type.
+ *
+ * This class provides a convenient way to interact with the MicroPython map type.
+ * Objects of this type do not own the map, and are not responsible for its lifetime.
+ */
+template <class Key, class Value>
+class MPyMapView {
+    static_assert(std::is_pointer<Value>::value, "upywrap::MpyMapView: Value must be a pointer type.");
+    static_assert(std::is_same<Key, qstr>::value, "upywrap::MPyMapView: Currently only qstr keys are supported.");
+
+public:
+    using mapped_type = Value;
+    explicit MPyMapView(mp_map_t* map_ptr) : map_ { map_ptr } {}
+
+    Value& operator[](const qstr key) {
+        mp_map_elem_t* elem = mp_map_lookup(map_, MP_OBJ_NEW_QSTR(key), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND);
+        return reinterpret_cast<Value&>(elem->value);
+    }
+
+    bool contains(const qstr key) const {
+        mp_map_elem_t* elem = mp_map_lookup(map_, MP_OBJ_NEW_QSTR(key), MP_MAP_LOOKUP);
+        return elem != nullptr;
+    }
+
+private:
+    mp_map_t* map_;
+};
+} // namespace upywrap

mpy_wrap_root_pointer.c

@@ -0,0 +1,7 @@
+//Add micropython-wrap as a "USERMOD" in micropython to register this root pointer
+//Alternative, this line can also be copy-pasted to user code.
+//Only required when using the `UPYWRAP_STATICPYOBJ_USE_ROOTPTR` configuration option (opt-in):
+//Needs to be done once: Add global type map for micropython-wrap to the micropython state as root pointer
+//This enables micropython-wrap to store anonymous type information while preventing visibility of the _StaticPyObjectStore
+//in user modules.
+MP_REGISTER_ROOT_POINTER(mp_map_t* micropython_wrap_global_storage);