bpo-43574 + bpo-39829: Optimize allocations

python · methane · Mar 14, 2022 · Mar 8, 2022 · Mar 8, 2022 · Mar 8, 2022
commit 37382e84ea003e73d73c4614ef1d93011f4dec42
diff --git a/Objects/listobject.c b/Objects/listobject.c
@@ -74,8 +74,9 @@ list_resize(PyListObject *self, Py_ssize_t newsize)
     if (newsize - Py_SIZE(self) > (Py_ssize_t)(new_allocated - newsize))
         new_allocated = ((size_t)newsize + 3) & ~(size_t)3;
 
-    if (newsize == 0)
-        new_allocated = 0;
+    /* Don't overallocate for lists that start empty or are set to empty. */
+    if (newsize == 0 || Py_SIZE(self) == 0)
+        new_allocated = newsize;
     num_allocated_bytes = new_allocated * sizeof(PyObject *);
     items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes);
     if (items == NULL) {
@@ -218,7 +219,6 @@ valid_index(Py_ssize_t i, Py_ssize_t limit)
 {
     /* The cast to size_t lets us use just a single comparison
        to check whether i is in the range: 0 <= i < limit.
-
        See:  Section 14.2 "Bounds Checking" in the Agner Fog
        optimization manual found at:
        https://www.agner.org/optimize/optimizing_cpp.pdf
@@ -787,11 +787,9 @@ list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v)
 
 /*[clinic input]
 list.insert
-
     index: Py_ssize_t
     object: object
     /
-
 Insert object before index.
 [clinic start generated code]*/
 
@@ -806,7 +804,6 @@ list_insert_impl(PyListObject *self, Py_ssize_t index, PyObject *object)
 
 /*[clinic input]
 list.clear
-
 Remove all items from list.
 [clinic start generated code]*/
 
@@ -820,7 +817,6 @@ list_clear_impl(PyListObject *self)
 
 /*[clinic input]
 list.copy
-
 Return a shallow copy of the list.
 [clinic start generated code]*/
 
@@ -833,10 +829,8 @@ list_copy_impl(PyListObject *self)
 
 /*[clinic input]
 list.append
-
      object: object
      /
-
 Append object to the end of the list.
 [clinic start generated code]*/
 
@@ -851,10 +845,8 @@ list_append(PyListObject *self, PyObject *object)
 
 /*[clinic input]
 list.extend
-
      iterable: object
      /
-
 Extend list by appending elements from the iterable.
10000
 [clinic start generated code]*/
 
@@ -865,7 +857,6 @@ list_extend(PyListObject *self, PyObject *iterable)
     PyObject *it;      /* iter(v) */
     Py_ssize_t m;                  /* size of self */
     Py_ssize_t n;                  /* guess for size of iterable */
-    Py_ssize_t mn;                 /* m + n */
     Py_ssize_t i;
     PyObject *(*iternext)(PyObject *);
 
@@ -889,13 +880,7 @@ list_extend(PyListObject *self, PyObject *iterable)
         /* It should not be possible to allocate a list large enough to cause
         an overflow on any relevant platform */
         assert(m < PY_SSIZE_T_MAX - n);
-        if (n && self->ob_item == NULL) {
-            if (list_preallocate_exact(self, n) < 0) {
-                Py_DECREF(iterable);
-                return NULL;
-            }
-        }
-        else if (list_resize(self, m + n) < 0) {
+        if (list_resize(self, m + n) < 0) {
             Py_DECREF(iterable);
             return NULL;
         }
@@ -935,16 +920,13 @@ list_extend(PyListObject *self, PyObject *iterable)
          */
     }
     else {
-        if (n && self->ob_item == NULL) {
+        /* Make room. */
+        if (self->ob_item == NULL) {
             if (list_preallocate_exact(self, n) < 0)
                 goto error;
         }
-        else {
-            mn = m + n;
-            /* Make room. */
-            if (list_resize(self, mn) < 0)
-                goto error;
-        }
+        else if (list_resize(self, m + n) < 0)
+            goto error;
         /* Make the list sane again. */
         Py_SET_SIZE(self, m);
     }
@@ -1009,12 +991,9 @@ list_inplace_concat(PyListObject *self, PyObject *other)
 
 /*[clinic input]
 list.pop
-
     index: Py_ssize_t = -1
     /
-
 Remove and return item at index (default last).
-
 Raises IndexError if list is empty or index is out of range.
 [clinic start generated code]*/
 
@@ -1301,19 +1280,14 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start)
 /*
 Return the length of the run beginning at lo, in the slice [lo, hi).  lo < hi
 is required on entry.  "A run" is the longest ascending sequence, with
-
     lo[0] <= lo[1] <= lo[2] <= ...
-
 or the longest descending sequence, with
-
     lo[0] > lo[1] > lo[2] > ...
-
 Boolean *descending is set to 0 in the former case, or to 1 in the latter.
 For its intended use in a stable mergesort, the strictness of the defn of
 "descending" is needed so that the caller can safely reverse a descending
 sequence without violating stability (strict > ensures there are no equal
 elements to get out of order).
-
 Returns -1 in case of error.
 */
 static Py_ssize_t
@@ -1355,20 +1329,14 @@ Locate the proper position of key in a sorted vector; if the vector contains
 an element equal to key, return the position immediately to the left of
 the leftmost equal element.  [gallop_right() does the same except returns
 the position to the right of the rightmost equal element (if any).]
-
 "a" is a sorted vector with n elements, starting at a[0].  n must be > 0.
-
 "hint" is an index at which to begin the search, 0 <= hint < n.  The closer
 hint is to the final result, the faster this runs.
-
 The return value is the int k in 0..n such that
-
     a[k-1] < key <= a[k]
-
 pretending that *(a-1) is minus infinity and a[n] is plus infinity.  IOW,
 key belongs at index k; or, IOW, the first k elements of a should precede
 key, and the last n-k should follow key.
-
 Returns -1 on error.  See listsort.txt for info on the method.
 */
 static Py_ssize_t
@@ -1449,13 +1417,9 @@ gallop_left(MergeState *ms, PyObject *key, PyObject **a, Py_ssize_t n, Py_ssize_
 /*
 Exactly like gallop_left(), except that if key already exists in a[0:n],
 finds the position immediately to the right of the rightmost equal value.
-
 The return value is the int k in 0..n such that
-
     a[k-1] <= key < a[k]
-
 or -1 if error.
-
 The code duplication is massive, but this is enough different given that
 we're sticking to "<" comparisons that it's much harder to follow if
 written as one routine with yet another "left or right?" flag.
@@ -2286,19 +2250,14 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms)
  */
 /*[clinic input]
 list.sort
-
     *
     key as keyfunc: object = None
     reverse: bool(accept={int}) = False
-
 Sort the list in ascending order and return None.
-
 The sort is in-place (i.e. the list itself is modified) and stable (i.e. the
 order of two equal elements is maintained).
-
 If a key function is given, apply it once to each list item and sort them,
 ascending or descending, according to their function values.
-
 The reverse flag can be set to sort in descending order.
 [clinic start generated code]*/
 
@@ -2584,7 +2543,6 @@ PyList_Sort(PyObject *v)
 
 /*[clinic input]
 list.reverse
-
 Reverse *IN PLACE*.
 [clinic start generated code]*/
 
@@ -2623,14 +2581,11 @@ PyList_AsTuple(PyObject *v)
 
 /*[clinic input]
 list.index
-
     value: object
     start: slice_index(accept={int}) = 0
     stop: slice_index(accept={int}, c_default="PY_SSIZE_T_MAX") = sys.maxsize
     /
-
 Return first index of value.
-
 Raises ValueError if the value is not present.
 [clinic start generated code]*/
 
@@ -2667,10 +2622,8 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start,
 
 /*[clinic input]
 list.count
-
      value: object
      /
-
 Return number of occurrences of value.
 [clinic start generated code]*/
 
@@ -2700,12 +2653,9 @@ list_count(PyListObject *self, PyObject *value)
 
 /*[clinic input]
 list.remove
-
      value: object
      /
-
 Remove first occurrence of value.
-
 Raises ValueError if the value is not present.
 [clinic start generated code]*/
 
@@ -2801,12 +2751,9 @@ list_richcompare(PyObject *v, PyObject *w, int op)
 
 /*[clinic input]
 list.__init__
-
     iterable: object(c_default="NULL") = ()
     /
-
 Built-in mutable sequence.
-
 If no argument is given, the constructor creates a new empty list.
 The argument must be an iterable if specified.
 [clinic start generated code]*/
@@ -2826,6 +2773,19 @@ list___init___impl(PyListObject *self, PyObject *iterable)
         (void)_list_clear(self);
     }
     if (iterable != NULL) {
+        if (_PyObject_HasLen(iterable)) {
+            Py_ssize_t iter_len = PyObject_Size(iterable);
+            if (iter_len == -1) {
+                if (!PyErr_ExceptionMatches(PyExc_TypeError)) {
+                    return -1;
+                }
+                PyErr_Clear();
+            }
+            if (iter_len > 0 && self->ob_item == NULL
+                && list_preallocate_exact(self, iter_len)) {
+                return -1;
+            }
+        }
         PyObject *rv = list_extend(self, iterable);
         if (rv == NULL)
             return -1;
@@ -2862,7 +2822,6 @@ list_vectorcall(PyObject *type, PyObject * const*args,
 
 /*[clinic input]
 list.__sizeof__
-
 Return the size of the list in memory, in bytes.
 [clinic start generated code]*/
 
@@ -3390,7 +3349,6 @@ PyTypeObject PyListRevIter_Type = {
 
 /*[clinic input]
 list.__reversed__
-
 Return a reverse iterator over the list.
 [clinic start generated code]*/