Merge pull request #8339 from charris/backport-8335

charris · web-flow · commit 4876f24bbbbd · 2016-12-04T19:20:58.000-05:00
BUG: Fix ndarray.tofile large file corruption in append mode.
diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c
@@ -44,10 +44,21 @@ npy_fallocate(npy_intp nbytes, FILE * fp)
     if (nbytes < 16 * 1024 * 1024) {
         return 0;
     }
+
     /* btrfs can take a while to allocate making release worthwhile */
     NPY_BEGIN_ALLOW_THREADS;
-    r = fallocate(fileno(fp), 0, npy_ftell(fp), nbytes);
+    /*
+     * flush in case there might be some unexpected interactions between the
+     * fallocate call and unwritten data in the descriptor
+     */
+    fflush(fp);
+    /*
+     * the flag "1" (=FALLOC_FL_KEEP_SIZE) is needed for the case of files
+     * opened in append mode (issue #8329)
+     */
+    r = fallocate(fileno(fp), 1, npy_ftell(fp), nbytes);
     NPY_END_ALLOW_THREADS;
+
     /*
      * early exit on no space, other errors will also get found during fwrite
      */
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
@@ -3843,6 +3843,15 @@ def test_largish_file(self):
             f.seek(d.nbytes)
             d.tofile(f)
             assert_equal(os.path.getsize(self.filename), d.nbytes * 2)
+        # check append mode (gh-8329)
+        open(self.filename, "w").close() # delete file contents
+        with open(self.filename, "ab") as f:
+            d.tofile(f)
+        assert_array_equal(d, np.fromfile(self.filename))
+        with open(self.filename, "ab") as f:
+            d.tofile(f)
+        assert_equal(os.path.getsize(self.filename), d.nbytes * 2)
+
 
     def test_file_position_after_fromfile(self):
         # gh-4118
@@ -4106,7 +4115,7 @@ def test_int_shape(self):
         x = np.eye(3)
         if IS_PYPY:
             x.resize(3, refcheck=False)
-        else:    
+        else:
             x.resize(3)
         assert_array_equal(x, np.eye(3)[0,:])
 
@@ -4135,7 +4144,7 @@ def test_zeros_appended(self):
         x = np.eye(3)
         if IS_PYPY:
             x.resize(2, 3, 3, refcheck=False)
-        else:    
+        else:
             x.resize(2, 3, 3)
         assert_array_equal(x[0], np.eye(3))
         assert_array_equal(x[1], np.zeros((3, 3)))
