10000 Merge pull request #11962 from mfkasim91/limitrowloadtxt · rth/numpy@87c1fcd · GitHub
[go: up one dir, main page]

Skip to content

Commit 87c1fcd

Browse files
authored
Merge pull request numpy#11962 from mfkasim91/limitrowloadtxt
ENH: maximum lines of content to be read from numpy.loadtxt
2 parents 289df94 + 4577a69 commit 87c1fcd

File tree

3 files changed

+63
-2
lines changed

3 files changed

+63
-2
lines changed

doc/release/1.16.0-notes.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ C API changes
7676
New Features
7777
============
7878

79+
``max_rows`` keyword added for ``np.loadtxt``
80+
---------------------------------------------
81+
New keyword ``max_rows`` in `numpy.loadtxt` sets the maximum rows of the
82+
content to be read after ``skiprows``, as in `numpy.genfromtxt`.
83+
7984

8085
Improvements
8186
============

numpy/lib/npyio.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -775,7 +775,7 @@ def floatconv(x):
775775

776776
def loadtxt(fname, dtype=float, comments='#', delimiter=None,
777777
converters=None, skiprows=0, usecols=None, unpack=False,
778-
ndmin=0, encoding='bytes'):
778+
ndmin=0, encoding='bytes', max_rows=None):
779779
"""
780780
Load data from a text file.
781781
@@ -837,6 +837,11 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
837837
the system default is used. The default value is 'bytes'.
838838
839839
.. versionadded:: 1.14.0
840+
max_rows : int, optional
841+
Read `max_rows` lines of content after `skiprows` lines. The default
842+
is to read all the lines.
843+
844+
.. versionadded:: 1.16.0
840845
841846
Returns
842847
-------
@@ -1018,7 +1023,9 @@ def read_data(chunk_size):
10181023
10191024
"""
10201025
X = []
1021-
for i, line in enumerate(itertools.chain([first_line], fh)):
1026+
line_iter = itertools.chain([first_line], fh)
1027+
line_iter = itertools.islice(line_iter, max_rows)
1028+
for i, line in enumerate(line_iter):
10221029
vals = split_line(line)
10231030
if len(vals) == 0:
10241031
continue

numpy/lib/tests/test_io.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1068,6 +1068,55 @@ def test_binary_load(self):
10681068
x = [b'5,6,7,\xc3\x95scarscar', b'15,2,3,hello', b'20,2,3,\xc3\x95scar']
10691069
assert_array_equal(x, np.array(x, dtype="S"))
10701070

1071+
def test_max_rows(self):
1072+
c = TextIO()
1073+
c.write('1,2,3,5\n4,5,7,8\n2,1,4,5')
1074+
c.seek(0)
1075+
x = np.loadtxt(c, dtype=int, delimiter=',',
1076+
max_rows=1)
1077+
a = np.array([1, 2, 3, 5], int)
1078+
assert_array_equal(x, a)
1079+
1080+
def test_max_rows_with_skiprows(self):
1081+
c = TextIO()
1082+
c.write('comments\n1,2,3,5\n4,5,7,8\n2,1,4,5')
1083+
c.seek(0)
1084+
x = np.loadtxt(c, dtype=int, delimiter=',',
1085+
skiprows=1, max_rows=1)
1086+
a = np.array([1, 2, 3, 5], int)
1087+
assert_array_equal(x, a)
1088+
1089+
c = TextIO()
1090+
c.write('comment\n1,2,3,5\n4,5,7,8\n2,1,4,5')
1091+
c.seek(0)
1092+
x = np.loadtxt(c, dtype=int, delimiter=',',
1093+
skiprows=1, max_rows=2)
1094+
a = np.array([[1, 2, 3, 5], [4, 5, 7, 8]], int)
1095+
assert_array_equal(x, a)
1096+
1097+
def test_max_rows_with_read_continuation(self):
1098+
c = TextIO()
1099+
c.write('1,2,3,5\n4,5,7,8\n2,1,4,5')
1100+
c.seek(0)
1101+
x = np.loadtxt(c, dtype=int, delimiter=',',
1102+
max_rows=2)
1103+
a = np.array([[1, 2, 3, 5], [4, 5, 7, 8]], int)
1104+
assert_array_equal(x, a)
1105+
# test continuation
1106+
x = np.loadtxt(c, dtype=int, delimiter=',')
1107+
a = np.array([2,1,4,5], int)
1108+
assert_array_equal(x, a)
1109+
1110+
def test_max_rows_larger(self):
1111+
#test max_rows > num rows
1112+
c = TextIO()
1113+
c.write('comment\n1,2,3,5\n4,5,7,8\n2,1,4,5')
1114+
c.seek(0)
1115+
x = np.loadtxt(c, dtype=int, delimiter=',',
1116+
skiprows=1, max_rows=6)
1117+
a = np.array([[1, 2, 3, 5], [4, 5, 7, 8], [2, 1, 4, 5]], int)
1118+
assert_array_equal(x, a)
1119+
10711120
class Testfromregex(object):
10721121
def test_record(self):
10731122
c = TextIO()

0 commit comments

Comments
 (0)
0