Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 7602879

Browse files
committedApr 17, 2018
Improve list() pre-sizing for inputs with known lengths
The list() constructor isn't taking full advantage of known input lengths or length hints. This commit makes the constructor pre-size and not over-allocate when the input size is known or can be reasonably estimated. A new test is added to test_list.py and a test needed to be modified in test_descr.py as it assumed that there is only one call to __length_hint__() in the list constructor.
1 parent 8f37e84 commit 7602879

File tree

4 files changed

+45
-1
lines changed

4 files changed

+45
-1
lines changed
 

‎Lib/test/test_descr.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -2028,7 +2028,7 @@ class X(Checker):
20282028
setattr(X, attr, obj)
20292029
setattr(X, name, SpecialDescr(meth_impl))
20302030
runner(X())
2031-
self.assertEqual(record, [1], name)
2031+
self.assertGreaterEqual(record.count(1), 1, name)
20322032

20332033
class X(Checker):
20342034
pass

‎Lib/test/test_list.py

+9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import sys
22
from test import list_tests
3+
from test.support import cpython_only
34
import pickle
45
import unittest
56

@@ -157,5 +158,13 @@ class L(list): pass
157158
with self.assertRaises(TypeError):
158159
(3,) + L([1,2])
159160

161+
@cpython_only
162+
def test_preallocation(self):
163+
iterable = [0] * 10
164+
iter_size = sys.getsizeof(iterable)
165+
166+
self.assertEqual(iter_size, sys.getsizeof(list([0] * 10)))
167+
self.assertEqual(iter_size, sys.getsizeof(list(range(10))))
168+
160169
if __name__ == "__main__":
161170
unittest.main()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
The list constructor will pre-size and not over-allocate when the input size
2+
is known or can be reasonably estimated.

‎Objects/listobject.c

+33
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,32 @@ list_resize(PyListObject *self, Py_ssize_t newsize)
7676
return 0;
7777
}
7878

79+
static int
80+
list_preallocate_exact(PyListObject *self, Py_ssize_t size)
81+
{
82+
PyObject **items;
83+
size_t allocated, num_allocated_bytes;
84+
85+
allocated = (size_t)size;
86+
if (allocated > (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) {
87+
PyErr_NoMemory();
88+
return -1;
89+
}
90+
91+
if (size == 0) {
92+
allocated = 0;
93+
}
94+
num_allocated_bytes = allocated * sizeof(PyObject *);
95+
items = (PyObject **)PyMem_Malloc(num_allocated_bytes);
96+
if (items == NULL) {
97+
PyErr_NoMemory();
98+
return -1;
99+
}
100+
self->ob_item = items;
101+
self->allocated = allocated;
102+
return 0;
103+
}
104+
79105
/* Debug statistic to compare allocations with reuse through the free list */
80106
#undef SHOW_ALLOC_COUNT
81107
#ifdef SHOW_ALLOC_COUNT
@@ -2649,6 +2675,13 @@ list___init___impl(PyListObject *self, PyObject *iterable)
26492675
(void)_list_clear(self);
26502676
}
26512677
if (iterable != NULL) {
2678+
Py_ssize_t iter_len = PyObject_LengthHint(iterable, 0);
2679+
if (iter_len == -1){
2680+
PyErr_Clear();
2681+
}
2682+
if (iter_len > 0 && list_preallocate_exact(self, iter_len)) {
2683+
return -1;
2684+
}
26522685
PyObject *rv = list_extend(self, iterable);
26532686
if (rv == NULL)
26542687
return -1;

0 commit comments

Comments
 (0)
Please sign in to comment.