gh-153946: Defer automatic GC in object graph builders

Author: pablogsal

Include/internal/pycore_gc.h

@@ -330,6 +330,14 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
 
 // Functions to clear types free lists
 extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
+
+// These calls nest across all threads in an interpreter. Allocation counters
+// continue advancing, explicit collections remain enabled, and a collection
+// that becomes due remains eligible at the next normal scheduling opportunity.
+// An automatic collection already starting concurrently may emit callbacks,
+// but the free-threaded collector will not traverse the protected object graph.
+PyAPI_FUNC(void) _PyGC_DeferAutomaticCollection(PyThreadState *tstate);
+PyAPI_FUNC(void) _PyGC_ResumeAutomaticCollection(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);
 
 union _PyStackRef;

Include/internal/pycore_interp_structs.h

@@ -224,6 +224,7 @@ struct gc_stats {
 struct _gc_runtime_state {
     /* Is automatic collection enabled? */
     int enabled;
+    int automatic_collection_pause_count;
     int debug;
     /* linked lists of container objects */
 #ifndef Py_GIL_DISABLED

Lib/test/test_gc.py

@@ -91,6 +91,80 @@ def __tp_del__(self):
 ###############################################################################
 
 class GCTests(unittest.TestCase):
+    @staticmethod
+    def total_collections():
+        return sum(stats["collections"] for stats in gc.get_stats())
+
+    @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
+    def test_defer_automatic_collection(self):
+        was_enabled = gc.isenabled()
+        gc.enable()
+        try:
+            with gc_threshold(1, 0, 0):
+                _testinternalcapi.defer_automatic_gc()
+                try:
+                    before = self.total_collections()
+                    objects = [[] for _ in range(10_000)]
+                    self.assertEqual(self.total_collections(), before)
+                    self.assertTrue(gc.isenabled())
+
+                    gc.collect()
+                    self.assertEqual(self.total_collections(), before + 1)
+                finally:
+                    _testinternalcapi.resume_automatic_gc()
+        finally:
+            if not was_enabled:
+                gc.disable()
+
+    @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
+    def test_defer_automatic_collection_nested(self):
+        was_enabled = gc.isenabled()
+        gc.enable()
+        try:
+            with gc_threshold(1, 0, 0):
+                _testinternalcapi.defer_automatic_gc()
+                try:
+                    _testinternalcapi.defer_automatic_gc()
+                    try:
+                        before = self.total_collections()
+                        objects = [[] for _ in range(10_000)]
+                        self.assertEqual(self.total_collections(), before)
+                    finally:
+                        _testinternalcapi.resume_automatic_gc()
+
+                    objects.extend([] for _ in range(10_000))
+                    self.assertEqual(self.total_collections(), before)
+                finally:
+                    _testinternalcapi.resume_automatic_gc()
+
+                objects.extend([] for _ in range(10_000))
+                self.assertGreater(self.total_collections(), before)
+        finally:
+            if not was_enabled:
+                gc.disable()
+
+    @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
+    def test_defer_automatic_collection_across_threads(self):
+        was_enabled = gc.isenabled()
+        gc.enable()
+        try:
+            with gc_threshold(1, 0, 0):
+                _testinternalcapi.defer_automatic_gc()
+                try:
+                    before = self.total_collections()
+                    objects = []
+                    thread = threading.Thread(
+                        target=lambda: objects.extend(
+                            [] for _ in range(10_000)))
+                    thread.start()
+                    thread.join()
+                    self.assertEqual(self.total_collections(), before)
+                finally:
+                    _testinternalcapi.resume_automatic_gc()
+        finally:
+            if not was_enabled:
+                gc.disable()
+
     def test_list(self):
         l = []
         l.append(l)

Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-00-00.gh-issue-153946.rst

@@ -0,0 +1,4 @@
+Add a private, nestable API for temporarily deferring automatic garbage
+collection without changing the interpreter-visible garbage collector state.
+Use it to avoid unproductive collections while converting ASTs to Python
+objects and unmarshalling objects from memory.

Modules/_testinternalcapi.c

@@ -2927,6 +2927,20 @@ get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored))
     return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size);
 }
 
+static PyObject *
+defer_automatic_gc(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    _PyGC_DeferAutomaticCollection(_PyThreadState_GET());
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+resume_automatic_gc(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    _PyGC_ResumeAutomaticCollection(_PyThreadState_GET());
+    Py_RETURN_NONE;
+}
+
 static PyObject *
 is_static_immortal(PyObject *self, PyObject *op)
 {
@@ -3378,6 +3392,8 @@ static PyMethodDef module_functions[] = {
     {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS},
     {"has_deferred_refcount", has_deferred_refcount, METH_O},
     {"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS},
+    {"defer_automatic_gc", defer_automatic_gc, METH_NOARGS},
+    {"resume_automatic_gc", resume_automatic_gc, METH_NOARGS},
     {"is_static_immortal", is_static_immortal, METH_O},
     {"incref_decref_delayed", incref_decref_delayed, METH_O},
     GET_NEXT_DICT_KEYS_VERSION_METHODDEF

Parser/asdl_c.py

@@ -2117,7 +2117,11 @@ class PartingShots(StaticVisitor):
     if (state == NULL) {
         return NULL;
     }
+    PyThreadState *tstate = _PyThreadState_GET();
+    // The new objects cannot be cyclic until the completed tree is returned.
+    _PyGC_DeferAutomaticCollection(tstate);
     PyObject *result = ast2obj_mod(state, t);
+    _PyGC_ResumeAutomaticCollection(tstate);
 
     return result;
 }
@@ -2254,6 +2258,7 @@ def generate_module_def(mod, metadata, f, internal_h):
         #include "pycore_ast.h"
         #include "pycore_ast_state.h"     // struct ast_state
         #include "pycore_ceval.h"         // _Py_EnterRecursiveCall()
+        #include "pycore_gc.h"            // _PyGC_DeferAutomaticCollection()
         #include "pycore_lock.h"          // _PyOnceFlag
         #include "pycore_modsupport.h"    // _PyArg_NoPositional()
         #include "pycore_pystate.h"       // _PyInterpreterState_GET()

Python/Python-ast.c

@@ -1783,6 +1783,26 @@ PyGC_IsEnabled(void)
     return gcstate->enabled;
 }
 
+void
+_PyGC_DeferAutomaticCollection(PyThreadState *tstate)
+{
+    GCState *gcstate = &tstate->interp->gc;
+    int previous = _Py_atomic_add_int(
+        &gcstate->automatic_collection_pause_count, 1);
+    (void)previous;
+    assert(previous >= 0);
+}
+
+void
+_PyGC_ResumeAutomaticCollection(PyThreadState *tstate)
+{
+    GCState *gcstate = &tstate->interp->gc;
+    int previous = _Py_atomic_add_int(
+        &gcstate->automatic_collection_pause_count, -1);
+    (void)previous;
+    assert(previous > 0);
+}
+
 /* Public API to invoke gc.collect() from C */
 Py_ssize_t
 PyGC_Collect(void)
@@ -1984,6 +2004,8 @@ _PyObject_GC_Link(PyObject *op)
     gcstate->generations[0].count++; /* number of allocated GC objects */
     if (gcstate->generations[0].count > gcstate->generations[0].threshold &&
         gcstate->enabled &&
+        !_Py_atomic_load_int_relaxed(
+            &gcstate->automatic_collection_pause_count) &&
         gcstate->generations[0].threshold &&
         !_Py_atomic_load_int_relaxed(&gcstate->collecting) &&
         !_PyErr_Occurred(tstate))
@@ -1996,7 +2018,9 @@ void
 _Py_RunGC(PyThreadState *tstate)
 {
     GCState *gcstate = get_gc_state();
-    if (!gcstate->enabled) {
+    if (!gcstate->enabled ||
+        _Py_atomic_load_int_relaxed(
+            &gcstate->automatic_collection_pause_count)) {
         return;
     }
     gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP);

Python/gc.c

@@ -2002,7 +2002,9 @@ gc_should_collect(GCState *gcstate)
     int count = _Py_atomic_load_int_relaxed(&gcstate->young.count);
     int threshold = gcstate->young.threshold;
     int gc_enabled = _Py_atomic_load_int_relaxed(&gcstate->enabled);
-    if (count <= threshold || threshold == 0 || !gc_enabled) {
+    int pause_count = _Py_atomic_load_int_relaxed(
+        &gcstate->automatic_collection_pause_count);
+    if (count <= threshold || threshold == 0 || !gc_enabled || pause_count) {
         return false;
     }
     if (gcstate->old[0].threshold == 0) {
@@ -2065,11 +2067,21 @@ record_deallocation(PyThreadState *tstate)
     }
 }
 
-static void
-gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, int generation)
+static bool
+gc_collect_internal(PyInterpreterState *interp,
+                    struct collection_state *state, int generation)
 {
     _PyEval_StopTheWorld(interp);
 
+    // A concurrent deferral may begin after this collection has emitted its
+    // start notification, but it must take effect before any heap traversal.
+    if (state->reason == _Py_GC_REASON_HEAP &&
+        _Py_atomic_load_int(
+            &state->gcstate->automatic_collection_pause_count)) {
+        _PyEval_StartTheWorld(interp);
+        return false;
+    }
+
     // update collection and allocation counters
     if (generation+1 < NUM_GENERATIONS) {
         state->gcstate->old[generation].count += 1;
@@ -2114,7 +2126,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
         if (err < 0) {
             _PyEval_StartTheWorld(interp);
             PyErr_NoMemory();
-            return;
+            return true;
         }
     }
     #endif
@@ -2124,7 +2136,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
     if (err < 0) {
         _PyEval_StartTheWorld(interp);
         PyErr_NoMemory();
-        return;
+        return true;
     }
 
 #ifdef GC_DEBUG
@@ -2171,7 +2183,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
         cleanup_worklist(&state->wrcb_to_call);
         cleanup_worklist(&state->objs_to_decref);
         PyErr_NoMemory();
-        return;
+        return true;
     }
 
     // Call tp_clear on objects in the unreachable set. This will cause
@@ -2181,6 +2193,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
 
     // Append objects with legacy finalizers to the "gc.garbage" list.
     handle_legacy_finalizers(state);
+    return true;
 }
 
 static struct gc_generation_stats *
@@ -2258,7 +2271,17 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
         .reason = reason,
     };
 
-    gc_collect_internal(interp, &state, generation);
+    if (!gc_collect_internal(interp, &state, generation)) {
+        if (PyDTrace_GC_DONE_ENABLED()) {
+            PyDTrace_GC_DONE(0);
+        }
+        if (reason != _Py_GC_REASON_SHUTDOWN) {
+            invoke_gc_callback(tstate, "stop", generation, 0, 0, 0, 0.0);
+        }
+        gcstate->frame = NULL;
+        _Py_atomic_store_int(&gcstate->collecting, 0);
+        return 0;
+    }
 
     m = state.collected;
     n = state.uncollectable;
@@ -2549,6 +2572,26 @@ PyGC_IsEnabled(void)
     return _Py_atomic_load_int_relaxed(&gcstate->enabled);
 }
 
+void
+_PyGC_DeferAutomaticCollection(PyThreadState *tstate)
+{
+    GCState *gcstate = &tstate->interp->gc;
+    int previous = _Py_atomic_add_int(
+        &gcstate->automatic_collection_pause_count, 1);
+    (void)previous;
+    assert(previous >= 0);
+}
+
+void
+_PyGC_ResumeAutomaticCollection(PyThreadState *tstate)
+{
+    GCState *gcstate = &tstate->interp->gc;
+    int previous = _Py_atomic_add_int(
+        &gcstate->automatic_collection_pause_count, -1);
+    (void)previous;
+    assert(previous > 0);
+}
+
 /* Public API to invoke gc.collect() from C */
 Py_ssize_t
 PyGC_Collect(void)

Python/gc_free_threading.c

@@ -9,6 +9,7 @@
 #include "Python.h"
 #include "pycore_call.h"             // _PyObject_CallNoArgs()
 #include "pycore_code.h"             // _PyCode_New()
+#include "pycore_gc.h"               // _PyGC_DeferAutomaticCollection()
 #include "pycore_hashtable.h"        // _Py_hashtable_t
 #include "pycore_long.h"             // _PyLong_IsZero()
 #include "pycore_object.h"           // _PyObject_IsUniquelyReferenced
@@ -1772,12 +1773,19 @@ read_object(RFILE *p)
         if (PySys_Audit("marshal.loads", "y#", p->ptr, (Py_ssize_t)(p->end - p->ptr)) < 0) {
             return NULL;
         }
+        PyThreadState *tstate = _PyThreadState_GET();
+        _PyGC_DeferAutomaticCollection(tstate);
+        v = r_object(p);
+        _PyGC_ResumeAutomaticCollection(tstate);
     } else if (p->fp || p->readable) {
         if (PySys_Audit("marshal.load", NULL) < 0) {
             return NULL;
         }
+        v = r_object(p);
+    }
+    else {
+        v = r_object(p);
     }
-    v = r_object(p);
     if (v == NULL && !PyErr_Occurred())
         PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
     return v;