micropython · projectgus · Nov 21, 2023
diff --git a/docs/library/esp32.rst b/docs/library/esp32.rst
@@ -76,6 +76,7 @@ Functions
        The "max new split" value in :func:`micropython.mem_info()` output
        corresponds to the largest free block of ESP-IDF heap that could be
        automatically added on demand to the MicroPython heap.
+       See also :func:`micropython.heap_sys_reserve`.
 
        The result of :func:`gc.mem_free()` is the total of the current "free"
        and "max new split" values printed by :func:`micropython.mem_info()`.

diff --git a/docs/library/micropython.rst b/docs/library/micropython.rst
@@ -102,6 +102,28 @@ Functions
    Note: `heap_locked()` is not enabled on most ports by default,
    requires ``MICROPY_PY_MICROPYTHON_HEAP_LOCKED``.
 
+.. function:: heap_sys_reserve([new_value])
+
+   .. note:: This function is only present on ports using the "auto split heap"
+      feature.
+
+   Get or set the number of bytes of free heap RAM that MicroPython *tries* to
+   reserve for the system.
+
+   When called with no arguments, returns the currently set value. When called
+   with an argument, updates the currently set value.
+
+   This is a soft limit that prevents "greedily" growing the MicroPython heap
+   too large. If MicroPython has no choice but to grow the heap or fail then
+   it will still grow the MicroPython heap beyond this limit.
+
+   Setting a higher value may help if the system is failing to allocate memory
+   outside MicroPython. Setting a lower value or even zero may help if
+   MicroPython memory is becoming unnecessarily fragmented.
+
+   Changing this limit cannot resolve memory issues that are caused by requiring
+   more memory than is physically available in the system.
+
 .. function:: kbd_intr(chr)
 
    Set the character that will raise a `KeyboardInterrupt` exception.  By

diff --git a/ports/esp32/gccollect.c b/ports/esp32/gccollect.c
@@ -84,4 +84,8 @@ size_t gc_get_max_new_split(void) {
     return heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT);
 }
 
+size_t gc_get_total_free(void) {
+    return heap_caps_get_free_size(MALLOC_CAP_DEFAULT);
+}
+
 #endif
diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h
@@ -187,6 +187,15 @@ void *esp_native_code_commit(void *, size_t, void *);
 #define MP_PLAT_COMMIT_EXEC(buf, len, reloc) esp_native_code_commit(buf, len, reloc)
 #define MP_SSIZE_MAX (0x7fffffff)
 
+#ifndef MP_PLAT_DEFAULT_HEAP_SYS_RESERVE
+// Try to keep some heap free for ESP-IDF system, unless Python is completely out of memory
+//
+// The default here is particularly high, because it's enough memory to initialise Wi-Fi and
+// create a TLS connection. If the program is not using these things, or if the heap doesn't
+// grow until after those things are created, then this will fragment memory more than needed.
+#define MP_PLAT_DEFAULT_HEAP_SYS_RESERVE (80 * 1024)
+#endif
+
 #if MICROPY_PY_SOCKET_EVENTS
 #define MICROPY_PY_SOCKET_EVENTS_HANDLER extern void socket_events_handler(void); socket_events_handler();
 #else

diff --git a/py/gc.c b/py/gc.c
@@ -238,6 +238,9 @@ void gc_add(void *start, void *end) {
 }
 
 #if MICROPY_GC_SPLIT_HEAP_AUTO
+
+size_t gc_heap_sys_reserve = MP_PLAT_DEFAULT_HEAP_SYS_RESERVE;
+
 // Try to automatically add a heap area large enough to fulfill 'failed_alloc'.
 STATIC bool gc_try_add_heap(size_t failed_alloc) {
     // 'needed' is the size of a heap large enough to hold failed_alloc, with
@@ -249,55 +252,82 @@ STATIC bool gc_try_add_heap(size_t failed_alloc) {
     // rounding up of partial block sizes).
     size_t needed = failed_alloc + MAX(2048, failed_alloc * 13 / 512);
 
-    size_t avail = gc_get_max_new_split();
+    size_t max_new_split = gc_get_max_new_split();
 
     DEBUG_printf("gc_try_add_heap failed_alloc " UINT_FMT ", "
-        "needed " UINT_FMT ", avail " UINT_FMT " bytes \n",
+        "needed " UINT_FMT ", max_new_split " UINT_FMT " bytes \n",
         failed_alloc,
         needed,
-        avail);
+        max_new_split);
 
-    if (avail < needed) {
+    if (max_new_split < needed) {
         // Can't fit this allocation, or system heap has nearly run out anyway
         return false;
     }
 
-    // Deciding how much to grow the total heap by each time is tricky:
+    // Measure the total Python heap size
+    size_t total_heap = 0;
+    for (mp_state_mem_area_t *area = &MP_STATE_MEM(area);
+         area != NULL;
+         area = NEXT_AREA(area)) {
+        total_heap += area->gc_pool_end - area->gc_alloc_table_start;
+        total_heap += ALLOC_TABLE_GAP_BYTE + sizeof(mp_state_mem_area_t);
+    }
+
+    // Deciding how much to allocate for the new "split" heap is tricky:
     //
     // - Grow by too small amounts, leads to heap fragmentation issues.
     //
     // - Grow by too large amounts, may lead to system heap running out of
     //   space.
     //
-    // Currently, this implementation is:
-    //
-    // - At minimum, aim to double the total heap size each time we add a new
-    //   heap.  i.e. without any large single allocations, total size will be
-    //   64KB -> 128KB -> 256KB -> 512KB -> 1MB, etc
-    //
-    // - If the failed allocation is too large to fit in that size, the new
-    //   heap is made exactly large enough for that allocation. Future growth
-    //   will double the total heap size again.
+
+    // Start by choosing the current total Python heap size for the new heap.
+    // With no other constraints, the total Python heap size would double each
+    // time: i.e 64KB -> 128KB -> 256KB -> 512KB -> 1MB, etc. This avoids
+    // fragmentation where possible.
+
+    size_t new_heap_size = total_heap;
+
+    // If this "greedy" size will cut free system heap below
+    // gc_heap_sys_reserve then reduce it to conserve that limit.
     //
-    // - If the new heap won't fit in the available free space, add the largest
-    //   new heap that will fit (this may lead to failed system heap allocations
-    //   elsewhere, but some allocation will likely fail in this circumstance!)
-    size_t total_heap = 0;
-    for (mp_state_mem_area_t *area = &MP_STATE_MEM(area);
-         area != NULL;
-         area = NEXT_AREA(area)) {
-        total_heap += area->gc_pool_end - area->gc_alloc_table_start;
-        total_heap += ALLOC_TABLE_GAP_BYTE + sizeof(mp_state_mem_area_t);
+    // (gc_heap_sys_reserve still isn't a hard limit, if the only
+    // options are returning a MemoryError to Python or using up the reserved
+    // system heap space then MicroPython will use up the reserved system heap
+    // space.)
+
+    size_t total_free = gc_get_total_free();
+    if (total_free < gc_heap_sys_reserve) {
+        new_heap_size = needed;
+    } else if (total_free - new_heap_size < gc_heap_sys_reserve) {
+        new_heap_size = total_free - gc_heap_sys_reserve;
+    }
+
+    // If this size is smaller than the size 'needed' to avoid an immediate
+    // MemoryError, increase to this size so the current failing allocation
+    // can succeed.
+
+    if (new_heap_size < needed) {
+        new_heap_size = needed;
     }
 
-    DEBUG_printf("total_heap " UINT_FMT " bytes\n", total_heap);
+    // If this size won't fit in the largest free system heap block, decrease
+    // it so it will fit (note: due to the check earlier, we already know
+    // max_new_split is large enough to hold 'needed')
+
+    if (new_heap_size > max_new_split) {
+        new_heap_size = max_new_split;
+    }
 
-    size_t to_alloc = MIN(avail, MAX(total_heap, needed));
+    DEBUG_printf("total_heap " UINT_FMT " total_free "
+        UINT_FMT " TRY_RESERVE_SYSTEM_HEAP " UINT_FMT " bytes\n",
+        total_heap, total_free, gc_heap_sys_reserve);
 
-    mp_state_mem_area_t *new_heap = MP_PLAT_ALLOC_HEAP(to_alloc);
+    mp_state_mem_area_t *new_heap = MP_PLAT_ALLOC_HEAP(new_heap_size);
 
     DEBUG_printf("MP_PLAT_ALLOC_HEAP " UINT_FMT " = %p\n",
-        to_alloc, new_heap);
+        new_heap_size, new_heap);
 
     if (new_heap == NULL) {
         // This should only fail:
@@ -307,7 +337,7 @@ STATIC bool gc_try_add_heap(size_t failed_alloc) {
         return false;
     }
 
-    gc_add(new_heap, (void *)new_heap + to_alloc);
+    gc_add(new_heap, (void *)new_heap + new_heap_size);
 
     return true;
 }

diff --git a/py/gc.h b/py/gc.h
@@ -40,6 +40,10 @@ void gc_add(void *start, void *end);
 // Port must implement this function to return the maximum available block of
 // RAM to allocate a new heap area into using MP_PLAT_ALLOC_HEAP.
 size_t gc_get_max_new_split(void);
+// This function returns the total amount of free RAM available for heap.
+size_t gc_get_total_free(void);
+// Runtime tuneable "soft" limit for free system heap
+extern size_t gc_heap_sys_reserve;
 #endif // MICROPY_GC_SPLIT_HEAP_AUTO
 #endif // MICROPY_GC_SPLIT_HEAP
 

diff --git a/py/modmicropython.c b/py/modmicropython.c
@@ -144,6 +144,20 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_locked_obj, mp_micropython_
 #endif
 #endif
 
+#if MICROPY_GC_SPLIT_HEAP_AUTO
+STATIC mp_obj_t mp_micropython_heap_sys_reserve(size_t n_args, const mp_obj_t *args) {
+    if (n_args > 0) {
+        mp_int_t new = mp_obj_get_int(args[0]);
+        if (new < 0) {
+            mp_raise_ValueError(NULL);
+        }
+        gc_heap_sys_reserve = new;
+    }
+    return mp_obj_new_int(gc_heap_sys_reserve);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_heap_sys_reserve_obj, 0, 1, mp_micropython_heap_sys_reserve);
+#endif
+
 #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0)
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_alloc_emergency_exception_buf_obj, mp_alloc_emergency_exception_buf);
 #endif
@@ -196,6 +210,9 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = {
     #if MICROPY_PY_MICROPYTHON_HEAP_LOCKED
     { MP_ROM_QSTR(MP_QSTR_heap_locked), MP_ROM_PTR(&mp_micropython_heap_locked_obj) },
     #endif
+    #if MICROPY_GC_SPLIT_HEAP_AUTO
+    { MP_ROM_QSTR(MP_QSTR_heap_sys_reserve), MP_ROM_PTR(&mp_micropython_heap_sys_reserve_obj) },
+    #endif
     #endif
     #if MICROPY_KBD_EXCEPTION
     { MP_ROM_QSTR(MP_QSTR_kbd_intr), MP_ROM_PTR(&mp_micropython_kbd_intr_obj) },