8000 Unpacker: Change max_buffer_size to 100MiB (#391) · ossdev07/msgpack-python@c356035 · GitHub
[go: up one dir, main page]
Skip to content

Commit c356035

Browse files
methanemethane
authored
Unpacker: Change max_buffer_size to 100MiB (msgpack#391)
1 parent 5399f81 commit c356035

File tree

2 files changed

+36
-31
lines changed

2 files changed

+36
-31
lines changed

‎msgpack/_unpacker.pyx

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -234,41 +234,42 @@ cdef class Unpacker(object):
234234
(See also simplejson)
235235
236236
:param int max_buffer_size:
237-
Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
237+
Limits size of data waiting unpacked. 0 means system's INT_MAX.
238+
The default value is 100*1024*1024 (100MiB).
238239
Raises `BufferFull` exception when it is insufficient.
239240
You should set this parameter when unpacking data from untrusted source.
240241
241242
:param int max_str_len:
242243
Deprecated, use *max_buffer_size* instead.
243-
Limits max length of str. (default: max_buffer_size or 1024*1024)
244+
Limits max length of str. (default: max_buffer_size)
244245
245246
:param int max_bin_len:
246247
Deprecated, use *max_buffer_size* instead.
247-
Limits max length of bin. (default: max_buffer_size or 1024*1024)
248+
Limits max length of bin. (default: max_buffer_size)
248249
249250
:param int max_array_len:
250-
Limits max length of array. (default: max_buffer_size or 128*1024)
251+
Limits max length of array. (default: max_buffer_size)
251252
252253
:param int max_map_len:
253-
Limits max length of map. (default: max_buffer_size//2 or 32*1024)
254+
Limits max length of map. (default: max_buffer_size//2)
254255
255256
:param int max_ext_len:
256257
Deprecated, use *max_buffer_size* instead.
257-
Limits max size of ext type. (default: max_buffer_size or 1024*1024)
258+
Limits max size of ext type. (default: max_buffer_size)
258259
259260
:param str unicode_errors:
260261
Error handler used for decoding str type. (default: `'strict'`)
261262
262263
263264
Example of streaming deserialize from file-like object::
264265
265-
unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
266+
unpacker = Unpacker(file_like)
266267
for o in unpacker:
267268
process(o)
268269
269270
Example of streaming deserialize from socket::
270271
271-
unpacker = Unpacker(max_buffer_size=10*1024*1024)
272+
unpacker = Unpacker()
272273
while True:
273274
buf = sock.recv(1024**2)
274275
if not buf:
@@ -305,7 +306,7 @@ cdef class Unpacker(object):
305306
def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
306307
bint use_list=True, bint raw=False, bint strict_map_key=True,
307308
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
308-
unicode_errors=None, Py_ssize_t max_buffer_size=0,
309+
unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024,
309310
object ext_hook=ExtType,
310311
Py_ssize_t max_str_len=-1,
311312
Py_ssize_t max_bin_len=-1,
@@ -325,23 +326,24 @@ cdef class Unpacker(object):
325326
if not PyCallable_Check(self.file_like_read):
326327
raise TypeError("`file_like.read` must be a callable.")
327328

329+
if not max_buffer_size:
330+
max_buffer_size = INT_MAX
328331
if max_str_len == -1:
329-
max_str_len = max_buffer_size or 1024*1024
332+
max_str_len = max_buffer_size
330333
if max_bin_len == -1:
331-
max_bin_len = max_buffer_size or 1024*1024
334+
max_bin_len = max_buffer_size
332335
if max_array_len == -1:
333-
max_array_len = max_buffer_size or 128*1024
336+
max_array_len = max_buffer_size
334337
if max_map_len == -1:
335-
max_map_len = max_buffer_size//2 or 32*1024
338+
max_map_len = max_buffer_size//2
336339
if max_ext_len == -1:
337-
max_ext_len = max_buffer_size or 1024*1024
340+
max_ext_len = max_buffer_size
338341

339-
if not max_buffer_size:
340-
max_buffer_size = INT_MAX
341342
if read_size > max_buffer_size:
342343
raise ValueError("read_size should be less or equal to max_buffer_size")
343344
if not read_size:
344345
read_size = min(max_buffer_size, 1024**2)
346+
345347
self.max_buffer_size = max_buffer_size
346348
self.read_size = read_size
347349
self.buf = <char*>PyMem_Malloc(read_size)

‎msgpack/fallback.py

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -193,39 +193,40 @@ class Unpacker(object):
193193
contains invalid UTF-8 string.
194194
195195
:param int max_buffer_size:
196-
Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
196+
Limits size of data waiting unpacked. 0 means 2**32-1.
197+
The default value is 100*1024*1024 (100MiB).
197198
Raises `BufferFull` exception when it is insufficient.
198199
You should set this parameter when unpacking data from untrusted source.
199200
200201
:param int max_str_len:
201202
Deprecated, use *max_buffer_size* instead.
202-
Limits max length of str. (default: max_buffer_size or 1024*1024)
203+
Limits max length of str. (default: max_buffer_size)
203204
204205
:param int max_bin_len:
205206
Deprecated, use *max_buffer_size* instead.
206-
Limits max length of bin. (default: max_buffer_size or 1024*1024)
207+
Limits max length of bin. (default: max_buffer_size)
207208
208209
:param int max_array_len:
209210
Limits max length of array.
210-
(default: max_buffer_size or 128*1024)
211+
(default: max_buffer_size)
211212
212213
:param int max_map_len:
213214
Limits max length of map.
214-
(default: max_buffer_size//2 or 32*1024)
215+
(default: max_buffer_size//2)
215216
216217
:param int max_ext_len:
217218
Deprecated, use *max_buffer_size* instead.
218-
Limits max size of ext type. (default: max_buffer_size or 1024*1024)
219+
Limits max size of ext type. (default: max_buffer_size)
219220
220221
Example of streaming deserialize from file-like object::
221222
222-
unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
223+
unpacker = Unpacker(file_like)
223224
for o in unpacker:
224225
process(o)
225226
226227
Example of streaming deserialize from socket::
227228
228-
unpacker = Unpacker(max_buffer_size=10*1024*1024)
229+
unpacker = Unpacker(max_buffer_size)
229230
while True:
230231
buf = sock.recv(1024**2)
231232
if not buf:
@@ -252,7 +253,7 @@ def __init__(
252253
object_pairs_hook=None,
253254
list_hook=None,
254255
unicode_errors=None,
255-
max_buffer_size=0,
256+
max_buffer_size=100 * 1024 * 1024,
256257
ext_hook=ExtType,
257258
max_str_len=-1,
258259
max_bin_len=-1,
@@ -285,18 +286,20 @@ def __init__(
285286
# state, which _buf_checkpoint records.
286287
self._buf_checkpoint = 0
287288

289+
if not max_buffer_size:
290+
max_buffer_size = 2 ** 31 - 1
288291
if max_str_len == -1:
289-
max_str_len = max_buffer_size or 1024 * 1024
292+
max_str_len = max_buffer_size
290293
if max_bin_len == -1:
291-
max_bin_len = max_buffer_size or 1024 * 1024
294+
max_bin_len = max_buffer_size
292295
if max_array_len == -1:
293-
max_array_len = max_buffer_size or 128 * 1024
296+
max_array_len = max_buffer_size
294297
if max_map_len == -1:
295-
max_map_len = max_buffer_size // 2 or 32 * 1024
298+
max_map_len = max_buffer_size // 2
296299
if max_ext_len == -1:
297-
max_ext_len = max_buffer_size or 1024 * 1024
300+
max_ext_len = max_buffer_size
298301

299-
self._max_buffer_size = max_buffer_size or 2 ** 31 - 1
302+
self._max_buffer_size = max_buffer_size
300303
if read_size > self._max_buffer_size:
301304
raise ValueError("read_size must be smaller than max_buffer_size")
302305
self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)

0 commit comments

Comments
 (0)
0