8000 Add Packer.buffer() (#320) · loude/msgpack-python@9e210bf · GitHub
[go: up one dir, main page]
Skip to content

Commit 9e210bf

Browse files
methanemethane
authored
Add Packer.buffer() (msgpack#320)
1 parent a8b3e97 commit 9e210bf

File tree

8 files changed

+107
-20
lines changed

8 files changed

+107
-20
lines changed

‎ChangeLog.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ If you need to handle large data, you need to specify limits manually.
1818
Other changes
1919
--------------
2020

21+
Add ``Unpacker.getbuffer()`` method.
22+
2123

2224

2325
0.5.6

‎docs/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ doctest:
153153
"results in $(BUILDDIR)/doctest/output.txt."
154154

155155
serve: html
156-
cd _build/html && python3 -m http.server
156+
python3 -m http.server -d _build/html
157157

158158
zip: html
159159
cd _build/html && zip -r ../../../msgpack-doc.zip .

‎docs/advanced.rst

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
Advanced usage
2+
===============
3+
4+
Packer
5+
------
6+
7+
autoreset
8+
~~~~~~~~~
9+
10+
When you used ``autoreset=False`` option of :class:`~msgpack.Packer`,
11+
``pack()`` method doesn't return packed ``bytes``.
12+
13+
You can use :meth:`~msgpack.Packer.bytes` or :meth:`~msgpack.Packer.getbuffer` to
14+
get packed data.
15+
16+
``bytes()`` returns ``bytes`` object. ``getbuffer()`` returns some bytes-like
17+
object. It's concrete type is implement detail and it will be changed in future
18+
versions.
19+
20+
You can reduce temporary bytes object by using ``Unpacker.getbuffer()``.
21+
22+
.. code-block:: python
23+
24+
packer = Packer(use_bin_type=True, autoreset=False)
25+
26+
packer.pack([1, 2])
27+
packer.pack([3, 4])
28+
29+
with open('data.bin', 'wb') as f:
30+
f.write(packer.getbuffer())
31+
32+
packer.reset() # reset internal buffer

‎docs/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ language data exchange.
88
:maxdepth: 1
99

1010
api
11+
advanced

‎msgpack/_packer.pyx

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ cdef extern from "pack.h":
4141
int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
4242
int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
4343

44+
cdef extern from "buff_converter.h":
45+
object buff_to_buff(char *, Py_ssize_t)
46+
4447
cdef int DEFAULT_RECURSE_LIMIT=511
4548
cdef long long ITEM_LIMIT = (2**32)-1
4649

@@ -349,9 +352,16 @@ cdef class Packer(object):
349352
return buf
350353

351354
def reset(self):
352-
"""Clear internal buffer."""
355+
"""Reset internal buffer.
356+
357+
This method is usaful only when autoreset=False.
358+
"""
353359
self.pk.length = 0
354360

355361
def bytes(self):
356-
"""Return buffer content."""
362+
"""Return internal buffer contents as bytes object"""
357363
return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
364+
365+
def getbuffer(self):
366+
"""Return view of internal buffer."""
367+
return buff_to_buff(self.pk.buf, self.pk.length)

‎msgpack/buff_converter.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#include "Python.h"
2+
3+
/* cython does not support this preprocessor check => write it in raw C */
4+
#if PY_MAJOR_VERSION == 2
5+
static PyObject *
6+
buff_to_buff(char *buff, Py_ssize_t size)
7+
{
8+
return PyBuffer_FromMemory(buff, size);
9+
}
10+
11+
#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3)
12+
static PyObject *
13+
buff_to_buff(char *buff, Py_ssize_t size)
14+
{
15+
return PyMemoryView_FromMemory(buff, size, PyBUF_READ);
16+
}
17+
#else
18+
static PyObject *
19+
buff_to_buff(char *buff, Py_ssize_t size)
20+
{
21+
Py_buffer pybuf;
22+
if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) {
23+
return NULL;
24+
}
25+
26+
return PyMemoryView_FromBuffer(&pybuf);
27+
}
28+
#endif

‎msgpack/fallback.py

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -860,43 +860,35 @@ def pack(self, obj):
860860
except:
861861
self._buffer = StringIO() # force reset
862862
raise
863-
ret = self._buffer.getvalue()
864863
if self._autoreset:
864+
ret = self._buffer.getvalue()
865865
self._buffer = StringIO()
866-
elif USING_STRINGBUILDER:
867-
self._buffer = StringIO(ret)
868-
return ret
866+
return ret
869867

870868
def pack_map_pairs(self, pairs):
871869
self._pack_map_pairs(len(pairs), pairs)
872-
ret = self._buffer.getvalue()
873870
if self._autoreset:
871+
ret = self._buffer.getvalue()
874872
self._buffer = StringIO()
875-
elif USING_STRINGBUILDER:
876-
self._buffer = StringIO(ret)
877-
return ret
873+
return ret
878874

879875
def pack_array_header(self, n):
880876
if n >= 2**32:
881877
raise PackValueError
882878
self._pack_array_header(n)
883-
ret = self._buffer.getvalue()
884879
if self._autoreset:
880+
ret = self._buffer.getvalue()
885881
self._buffer = StringIO()
886-
elif USING_STRINGBUILDER:
887-
self._buffer = StringIO(ret)
888-
return ret
882+
return ret
889883

890884
def pack_map_header(self, n):
891885
if n >= 2**32:
892886
raise PackValueError
893887
self._pack_map_header(n)
894-
ret = self._buffer.getvalue()
895888
if self._autoreset:
889+
ret = self._buffer.getvalue()
896890
self._buffer = StringIO()
897-
elif USING_STRINGBUILDER:
898-
self._buffer = StringIO(ret)
899-
return ret
891+
return ret
900892

901893
def pack_ext_type(self, typecode, data):
902894
if not isinstance(typecode, int):
@@ -976,7 +968,19 @@ def _pack_bin_header(self, n):
976968
raise PackValueError('Bin is too large')
977969

978970
def bytes(self):
971+
"""Return internal buffer contents as bytes object"""
979972
return self._buffer.getvalue()
980973

981974
def reset(self):
975+
"""Reset internal buffer.
976+
977+
This method is usaful only when autoreset=False.
978+
"""
982979
self._buffer = StringIO()
980+
981+
def getbuffer(self):
982+
"""Return view of internal buffer."""
983+
if USING_STRINGBUILDER or not PY3:
984+
return memoryview(self.bytes())
985+
else:
986+
return self._buffer.getbuffer()

‎test/test_pack.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import struct
66
from pytest import raises, xfail
77

8-
from msgpack import packb, unpackb, Unpacker, Packer
8+
from msgpack import packb, unpackb, Unpacker, Packer, pack
99

1010
from collections import OrderedDict
1111
from io import BytesIO
@@ -148,3 +148,13 @@ def test_pairlist():
148148
packed = packer.pack_map_pairs(pairlist)
149149
unpacked = unpackb(packed, object_pairs_hook=list)
150150
assert pairlist == unpacked
151+
152+
def test_get_buffer():
153+
packer = Packer(autoreset=0, use_bin_type=True)
154+
packer.pack([1, 2])
155+
strm = BytesIO()
156+
strm.write(packer.getbuffer())
157+
written = strm.getvalue()
158+
159+
expected = packb([1, 2], use_bin_type=True)
160+
assert written == expected

0 commit comments

Comments
 (0)
0