multiprocess: don't crash on missing gauge_live/sum files (prometheus#424)

xavfernandez · brian-brazil · commit 04c112aca4cf · 2019-06-18T12:46:39.000+01:00
Fixes prometheus#425 Signed-off-by: Xavier Fernandez <xavier.fernandez@polyconseil.fr>
diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py
@@ -12,6 +12,11 @@
 from .samples import Sample
 from .utils import floatToGoString
 
+try:  # Python3
+    FileNotFoundError
+except NameError:  # Python >= 2.5
+    FileNotFoundError = IOError
+
 MP_METRIC_HELP = 'Multiprocess metric'
 
 
@@ -54,7 +59,16 @@ def _parse_key(key):
         for f in files:
             parts = os.path.basename(f).split('_')
             typ = parts[0]
-            for key, value, pos in MmapedDict.read_all_values_from_file(f):
+            try:
+                file_values = MmapedDict.read_all_values_from_file(f)
+            except FileNotFoundError:
+                if typ == 'gauge' and parts[1] in ('liveall', 'livesum'):
+                    # Those files can disappear between the glob of collect
+                    # and now (via a mark_process_dead call) so don't fail if
+                    # the file is missing
+                    continue
+                raise
+            for key, value, pos in file_values:
                 metric_name, name, labels, labels_key = _parse_key(key)
 
                 metric = metrics.get(metric_name)
diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py
@@ -270,6 +270,16 @@ def add_label(key, value):
 
         self.assertEqual(metrics['h'].samples, expected_histogram)
 
+    def test_missing_gauge_file_during_merge(self):
+        # These files don't exist, just like if mark_process_dead(9999999) had been
+        # called during self.collector.collect(), after the glob found it
+        # but before the merge actually happened.
+        # This should not raise and return no metrics
+        self.assertFalse(self.collector.merge([
+            os.path.join(self.tempdir, 'gauge_liveall_9999999.db'),
+            os.path.join(self.tempdir, 'gauge_livesum_9999999.db'),
+        ]))
+
 
 class TestMmapedDict(unittest.TestCase):
     def setUp(self):
