Avoid re-entrant calls to GC collector's callback (prometheus#343)

akx · brian-brazil · commit 33bc65ba9faa · 2018-11-16T11:40:56.000Z
It looks like the Python GC engine can call the GC callback
while it's still being called (likely if the callback does significant
enough work), which may, if you're unlucky enough, lead into a stack
overflow Python is unable to recover from.

This patch should fix it, though no empirical tests have been made.

Signed-off-by: Aarni Koskela &lt;akx@iki.fi&gt;
diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py
@@ -47,19 +47,32 @@ def __init__(self, registry=core.REGISTRY, gc=gc):
 
         times = {}
 
+        # Avoid _cb() being called re-entrantly
+        # by setting this flag and clearing it once
+        # the callback operation is complete.
+        # See https://github.com/prometheus/client_python/issues/322#issuecomment-438021132
+        self.gc_cb_active = False
+
         def _cb(phase, info):
-            gen = info['generation']
-
-            if phase == 'start':
-                times[gen] = time.time()
-
-            if phase == 'stop':
-                delta = time.time() - times[gen]
-                latency.labels(gen).observe(delta)
-                if 'collected' in info:
-                    collected.labels(gen).observe(info['collected'])
-                if 'uncollectable' in info:
-                    uncollectable.labels(gen).observe(info['uncollectable'])
+            try:
+                if self.gc_cb_active:
+                    return
+                self.gc_cb_active = True
+
+                gen = info['generation']
+
+                if phase == 'start':
+                    times[gen] = time.time()
+
+                if phase == 'stop':
+                    delta = time.time() - times[gen]
+                    latency.labels(gen).observe(delta)
+                    if 'collected' in info:
+                        collected.labels(gen).observe(info['collected'])
+                    if 'uncollectable' in info:
+                        uncollectable.labels(gen).observe(info['uncollectable'])
+            finally:
+                self.gc_cb_active = False
 
         gc.callbacks.append(_cb)
 
