From 8fb709f2e0438862020d8810fa70a81fb5dac7d4 Mon Sep 17 00:00:00 2001 From: Peter Fischer Date: Thu, 30 Jul 2020 16:48:51 +0200 Subject: [PATCH 001/105] Fix datetime before epoch on windows in cython implementation (#436) Cython implementation still used datetime.from_timestamp method, which does not work on windows. Update the cython implementation to use utc time and delta and add a regression test to highlight the issue. --- Makefile | 1 + msgpack/unpack.h | 34 +++++++++++++++++++++------------- test/test_timestamp.py | 8 ++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 2a4c0af8..0110ddfa 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ clean: rm -rf build rm -f msgpack/_cmsgpack.cpp rm -f msgpack/_cmsgpack.*.so + rm -f msgpack/_cmsgpack.*.pyd rm -rf msgpack/__pycache__ rm -rf test/__pycache__ diff --git a/msgpack/unpack.h b/msgpack/unpack.h index debdf715..868b96e7 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -341,7 +341,26 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos else if (u->timestamp == 0) { // Timestamp py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec); } - else { // float or datetime + else if (u->timestamp == 3) { // datetime + // Calculate datetime using epoch + delta + // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps + PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType); + if (epoch == NULL) { + return -1; + } + + PyObject* d = PyDelta_FromDSU(0, ts.tv_sec, ts.tv_nsec / 1000); + if (d == NULL) { + Py_DECREF(epoch); + return -1; + } + + py = PyNumber_Add(epoch, d); + + Py_DECREF(epoch); + Py_DECREF(d); + } + else { // float PyObject *a = PyFloat_FromDouble((double)ts.tv_nsec); if (a == NULL) return -1; @@ -358,18 +377,7 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos a = PyNumber_Add(b, c); Py_DECREF(b); Py_DECREF(c); - - if (u->timestamp == 3) { // datetime - PyObject *t = PyTuple_Pack(2, a, u->utc); - Py_DECREF(a); - if (t == NULL) { - return -1; - } - py = PyDateTime_FromTimestamp(t); - Py_DECREF(t); - } else { // float - py = a; - } + py = a; } } else { py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 823fe04e..edc488af 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -99,6 +99,14 @@ def test_unpack_datetime(): assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_pack_unpack_before_epoch(): + t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc) + packed = msgpack.packb(t_in, datetime=True) + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == t_in + + @pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime(): t = Timestamp(42, 14000) From 44bc2bd439808ad7563ef8a558ad6ccfe175a66a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 4 Dec 2020 17:23:09 +0900 Subject: [PATCH 002/105] Update docstring --- msgpack/_packer.pyx | 8 ++++- msgpack/_unpacker.pyx | 80 +++++++++++++++++-------------------------- msgpack/fallback.py | 25 +++++++++++++- 3 files changed, 63 insertions(+), 50 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index b4706463..e6cd2c7d 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -63,7 +63,7 @@ cdef class Packer(object): """ MessagePack Packer - usage:: + Usage:: packer = Packer() astream.write(packer.pack(a)) @@ -94,6 +94,12 @@ cdef class Packer(object): This is useful when trying to implement accurate serialization for python types. + :param bool datetime: + If set to true, datetime with tzinfo is packed into Timestamp type. + Note that the tzinfo is stripped in the timestamp. + You can get UTC datetime with `timestamp=3` option of the Unpacker. + (Python 2 is not supported). + :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 4340e044..e4f3f1e4 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -212,65 +212,49 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, cdef class Unpacker(object): - """Streaming unpacker. - - Arguments: - - :param file_like: - File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - - :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) - - :param bool use_list: - If true, unpack msgpack array to Python list. - Otherwise, unpack to Python tuple. (default: True) - - :param bool raw: - If true, unpack msgpack raw to Python bytes. - Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). + """ + MessagePack Packer - :param bool strict_map_key: - If true (default), only str or bytes are accepted for map (dict) keys. + Usage:: - :param callable object_hook: - When specified, it should be callable. - Unpacker calls it with a dict argument after unpacking msgpack map. - (See also simplejson) + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) - :param callable object_pairs_hook: - When specified, it should be callable. - Unpacker calls it with a list of key-value pairs after unpacking msgpack map. - (See also simplejson) + Packer's constructor has some keyword arguments: - :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX. - The default value is 100*1024*1024 (100MiB). - Raises `BufferFull` exception when it is insufficient. - You should set this parameter when unpacking data from untrusted source. + :param callable default: + Convert user type to builtin type that Packer supports. + See also simplejson's document. - :param int max_str_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size) + :param bool use_single_float: + Use single precision float type for float. (default: False) - :param int max_bin_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size) + :param bool autoreset: + Reset buffer after each pack and return its content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. - :param int max_array_len: - Limits max length of array. (default: max_buffer_size) + :param bool use_bin_type: + Use bin type introduced in msgpack spec 2.0 for bytes. + It also enables str8 type for unicode. (default: True) - :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2) + :param bool strict_types: + If set to true, types will be checked to be exact. Derived classes + from serializable types will not be serialized and will be + treated as unsupported type and forwarded to default. + Additionally tuples will not be serialized as lists. + This is useful when trying to implement accurate serialization + for python types. - :param int max_ext_len: - Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size) + :param bool datetime: + If set to true, datetime with tzinfo is packed into Timestamp type. + Note that the tzinfo is stripped in the timestamp. + You can get UTC datetime with `timestamp=3` option of the Unpacker. + (Python 2 is not supported). :param str unicode_errors: - Error handler used for decoding str type. (default: `'strict'`) - + The error handler for encoding unicode. (default: 'strict') + DO NOT USE THIS!! This option is kept for very specific usage. Example of streaming deserialize from file-like object:: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9739d53b..0bfa94ea 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -744,7 +744,7 @@ class Packer(object): """ MessagePack Packer - Usage: + Usage:: packer = Packer() astream.write(packer.pack(a)) @@ -784,6 +784,29 @@ class Packer(object): :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. + + Example of streaming deserialize from file-like object:: + + unpacker = Unpacker(file_like) + for o in unpacker: + process(o) + + Example of streaming deserialize from socket:: + + unpacker = Unpacker() + while True: + buf = sock.recv(1024**2) + if not buf: + break + unpacker.feed(buf) + for o in unpacker: + process(o) + + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``OutOfData`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. """ def __init__( From 2df517999b012737b1cf36f12bac13541a8eb6e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 13:39:24 +0900 Subject: [PATCH 003/105] Travis: Reduce build Save credits. --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5132b4c4..5980d630 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,17 @@ dist: xenial language: python cache: pip arch: - - amd64 - arm64 + python: # Available Python (PyPy) can be listed by: # # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - - "3.4" - - "3.5" - "3.6" - "3.7" - "3.8" - "3.9-dev" - _pure: &pure install: - pip install -U pip From 7d6b4dfb516451735150d3f796f0b35ed5f004a4 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg <65945052+tsahee@users.noreply.github.com> Date: Fri, 11 Dec 2020 07:30:49 +0200 Subject: [PATCH 004/105] Build arm64 wheels (#439) --- .github/workflows/linux.yml | 10 ++++++++++ Makefile | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 89bdb4e4..50a0abe7 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -30,6 +30,16 @@ jobs: run: | make linux-wheel + - name: Install qemu-user-static for docker + shell: bash + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build arm64 wheels + shell: bash + run: | + make linux-arm64-wheel + - name: Run test (3.8) run: | pip install pytest diff --git a/Makefile b/Makefile index 0110ddfa..05cca554 100644 --- a/Makefile +++ b/Makefile @@ -38,3 +38,7 @@ update-docker: linux-wheel: docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686 bash docker/buildwheel.sh docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh + +.PHONY: linux-arm64-wheel +linux-arm64-wheel: + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_aarch64 bash docker/buildwheel.sh From d893697eab07a2cf2a02f5115d4e7bae99d07e3f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 19:16:14 +0900 Subject: [PATCH 005/105] v1.0.1 --- ChangeLog.rst | 8 ++++++++ msgpack/_version.py | 2 +- setup.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index d922e847..bb3a6332 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.0.1 +===== + +* Add Python 3.9 and linux/arm64 wheels. (#439) +* Fixed Unpacker.tell() after read_bytes() (#426) +* Fixed unpacking datetime before epoch on Windows (#433) +* Fixed fallback Packer didn't check DateTime.tzinfo (#434) + 1.0.0 ===== diff --git a/msgpack/_version.py b/msgpack/_version.py index 9f55cf50..95e81294 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 0) +version = (1, 0, 1) diff --git a/setup.py b/setup.py index 2ec9ca70..8e88750a 100755 --- a/setup.py +++ b/setup.py @@ -132,6 +132,7 @@ def __init__(self, *args, **kwargs): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From edd56036616e5f7211d5cd840a4d00bb43a7f9ee Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 19:31:24 +0900 Subject: [PATCH 006/105] Actions: Add Python 3.9 --- .github/workflows/linux.yml | 11 +++++++++++ .github/workflows/mac.yml | 17 +++++++++++++++++ .github/workflows/windows.yaml | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 50a0abe7..f9707b1c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -47,6 +47,17 @@ jobs: pytest -v test + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: 3.9 + + - name: Run test (3.9) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index fb2c67f4..78d944cc 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -35,6 +35,23 @@ jobs: pytest -v test + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: "3.9" + + - name: Build wheel + run: | + pip install setuptools wheel + python setup.py bdist_wheel + + - name: Run test + run: | + pip install pytest + pip install -v msgpack --only-binary :all: -f dist/ --no-index + pytest -v test + + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index cecb8258..139a5a64 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -63,6 +63,20 @@ jobs: run: | ci/runtests.sh + - name: Python 3.9 (amd64) + env: + PYTHON: "py -3.9-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.9 (x86) + env: + PYTHON: "py -3.9-32" + shell: bash + run: | + ci/runtests.sh + - name: Upload Wheels uses: actions/upload-artifact@v1 with: From 8029f95516dbfddf2fea61efb06dc08ded84aab7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 19:33:20 +0900 Subject: [PATCH 007/105] Add Python 3.9 wheels --- docker/shared.env | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/shared.env b/docker/shared.env index 17abdd8f..3601a075 100644 --- a/docker/shared.env +++ b/docker/shared.env @@ -1,4 +1,5 @@ PYTHON_VERSIONS=( + cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m From 753b3706d80a7bc5a29147730804e867b97eee57 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 14:21:27 +0900 Subject: [PATCH 008/105] Fix overflow in unpacking timestamp to datetime (#452) --- msgpack/unpack.h | 6 +++--- test/test_timestamp.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 868b96e7..34212bcd 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -342,21 +342,21 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec); } else if (u->timestamp == 3) { // datetime - // Calculate datetime using epoch + delta + // Calculate datetime using epoch + delta // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType); if (epoch == NULL) { return -1; } - PyObject* d = PyDelta_FromDSU(0, ts.tv_sec, ts.tv_nsec / 1000); + PyObject* d = PyDelta_FromDSU(ts.tv_sec/(24*3600), ts.tv_sec%(24*3600), ts.tv_nsec / 1000); if (d == NULL) { Py_DECREF(epoch); return -1; } py = PyNumber_Add(epoch, d); - + Py_DECREF(epoch); Py_DECREF(d); } diff --git a/test/test_timestamp.py b/test/test_timestamp.py index edc488af..6a29be77 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -129,3 +129,14 @@ def test_pack_datetime(): assert x assert x[0] == dt assert msgpack.unpackb(packed) is None + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_issue451(): + # https://github.com/msgpack/msgpack-python/issues/451 + dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc) + packed = msgpack.packb(dt, datetime=True) + assert packed == b"\xd6\xff\xf4\x86eL" + + unpacked = msgpack.unpackb(packed, timestamp=3) + assert dt == unpacked From 94336cf914c202718e37f27d664517c9a8c79d50 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:03:05 +0900 Subject: [PATCH 009/105] Fix some travis builds. (#453) --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5980d630..4974d26c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ _pure: &pure matrix: include: - name: 32bit build + arch: amd64 language: python services: - docker @@ -58,10 +59,12 @@ matrix: <<: *pure - name: "pypy2.7" + arch: amd64 python: "pypy2.7-7.1.1" <<: *pure - name: "pypy3" + arch: amd64 python: "pypy3.6-7.1.1" <<: *pure From 051f9ded1fe2343ffdf55e7a5ec2da1dab7638b9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:13:35 +0900 Subject: [PATCH 010/105] format markdown --- README.md | 126 +++++++++++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index ac52d94b..2d5dd889 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,9 @@ See note below for detail. ## Install - - $ pip install msgpack - +``` +$ pip install msgpack +``` ### Pure Python implementation @@ -103,18 +103,18 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with `unpack` and `load` unpacks from a file-like object. ```pycon - >>> import msgpack - >>> msgpack.packb([1, 2, 3], use_bin_type=True) - '\x93\x01\x02\x03' - >>> msgpack.unpackb(_, raw=False) - [1, 2, 3] +>>> import msgpack +>>> msgpack.packb([1, 2, 3], use_bin_type=True) +'\x93\x01\x02\x03' +>>> msgpack.unpackb(_, raw=False) +[1, 2, 3] ``` `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: ```pycon - >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) - (1, 2, 3) +>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) +(1, 2, 3) ``` You should always specify the `use_list` keyword argument for backward compatibility. @@ -129,18 +129,18 @@ Read the docstring for other options. stream (or from bytes provided through its `feed` method). ```py - import msgpack - from io import BytesIO +import msgpack +from io import BytesIO - buf = BytesIO() - for i in range(100): - buf.write(msgpack.packb(i, use_bin_type=True)) +buf = BytesIO() +for i in range(100): + buf.write(msgpack.packb(i, use_bin_type=True)) - buf.seek(0) +buf.seek(0) - unpacker = msgpack.Unpacker(buf, raw=False) - for unpacked in unpacker: - print(unpacked) +unpacker = msgpack.Unpacker(buf, raw=False) +for unpacked in unpacker: + print(unpacked) ``` @@ -150,27 +150,27 @@ It is also possible to pack/unpack custom data types. Here is an example for `datetime.datetime`. ```py - import datetime - import msgpack +import datetime +import msgpack - useful_dict = { - "id": 1, - "created": datetime.datetime.now(), - } +useful_dict = { + "id": 1, + "created": datetime.datetime.now(), +} - def decode_datetime(obj): - if '__datetime__' in obj: - obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") - return obj +def decode_datetime(obj): + if '__datetime__' in obj: + obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") + return obj - def encode_datetime(obj): - if isinstance(obj, datetime.datetime): - return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} - return obj +def encode_datetime(obj): + if isinstance(obj, datetime.datetime): + return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} + return obj - packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) - this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) +packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) +this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) ``` `Unpacker`'s `object_hook` callback receives a dict; the @@ -183,25 +183,25 @@ key-value pairs. It is also possible to pack/unpack custom data types using the **ext** type. ```pycon - >>> import msgpack - >>> import array - >>> def default(obj): - ... if isinstance(obj, array.array) and obj.typecode == 'd': - ... return msgpack.ExtType(42, obj.tostring()) - ... raise TypeError("Unknown type: %r" % (obj,)) - ... - >>> def ext_hook(code, data): - ... if code == 42: - ... a = array.array('d') - ... a.fromstring(data) - ... return a - ... return ExtType(code, data) - ... - >>> data = array.array('d', [1.2, 3.4]) - >>> packed = msgpack.packb(data, default=default, use_bin_type=True) - >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) - >>> data == unpacked - True +>>> import msgpack +>>> import array +>>> def default(obj): +... if isinstance(obj, array.array) and obj.typecode == 'd': +... return msgpack.ExtType(42, obj.tostring()) +... raise TypeError("Unknown type: %r" % (obj,)) +... +>>> def ext_hook(code, data): +... if code == 42: +... a = array.array('d') +... a.fromstring(data) +... return a +... return ExtType(code, data) +... +>>> data = array.array('d', [1.2, 3.4]) +>>> packed = msgpack.packb(data, default=default, use_bin_type=True) +>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) +>>> data == unpacked +True ``` @@ -226,11 +226,11 @@ You can pack into and unpack from this old spec using `use_bin_type=False` and `raw=True` options. ```pycon - >>> import msgpack - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) - [b'spam', b'eggs'] - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) - [b'spam', 'eggs'] +>>> import msgpack +>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) +[b'spam', b'eggs'] +>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) +[b'spam', 'eggs'] ``` ### ext type @@ -238,10 +238,10 @@ and `raw=True` options. To use the **ext** type, pass `msgpack.ExtType` object to packer. ```pycon - >>> import msgpack - >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) - >>> msgpack.unpackb(packed) - ExtType(code=42, data='xyzzy') +>>> import msgpack +>>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) +>>> msgpack.unpackb(packed) +ExtType(code=42, data='xyzzy') ``` You can use it with `default` and `ext_hook`. See below. From f34fca7fb55e2c8adde39c34ac48542649a24d11 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:21:41 +0900 Subject: [PATCH 011/105] Update readme --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2d5dd889..d8ce9baa 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,10 @@ This package provides CPython bindings for reading and writing MessagePack data. ### PyPI package name -TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. -Do `pip uninstall msgpack-python; pip install -U msgpack` instead. +Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. -Package name on PyPI was changed to msgpack from 0.5. -I upload transitional package (msgpack-python 0.5 which depending on msgpack) -for smooth transition from msgpack-python to msgpack. - -Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-python`, -msgpack is removed, and `import msgpack` fail. +When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before +`pip install -U msgpack`. ### Compatibility with the old format From c0516c603f0eb6555117e312f5cdfb383853bc8e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:43:04 +0900 Subject: [PATCH 012/105] v1.0.2 --- ChangeLog.rst | 5 +++++ msgpack/_version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index bb3a6332..230cc301 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,8 @@ +1.0.2 +===== + +* Fix year 2038 problem regression in 1.0.1. (#451) + 1.0.1 ===== diff --git a/msgpack/_version.py b/msgpack/_version.py index 95e81294..1c83c8ed 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 1) +version = (1, 0, 2) From 431ef45c8ebff54c2b182dda46e7edb78a4e271b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 17:43:37 +0900 Subject: [PATCH 013/105] Use manylinux1 instead of manylinux2010 --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 05cca554..6f29aede 100644 --- a/Makefile +++ b/Makefile @@ -31,13 +31,13 @@ clean: .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux2010_i686 - docker pull quay.io/pypa/manylinux2010_x86_64 + docker pull quay.io/pypa/manylinux1_i686 + docker pull quay.io/pypa/manylinux1_x86_64 .PHONY: linux-wheel linux-wheel: - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686 bash docker/buildwheel.sh - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh .PHONY: linux-arm64-wheel linux-arm64-wheel: From 3b71818bb0e4cbafdb58895fa47704563448b6ac Mon Sep 17 00:00:00 2001 From: Guy Tuval Date: Sat, 2 Jan 2021 08:39:37 +0200 Subject: [PATCH 014/105] Refactor fallback read header (#441) --- msgpack/fallback.py | 247 +++++++++++++++----------------------------- 1 file changed, 82 insertions(+), 165 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0bfa94ea..7dbc67ad 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,5 +1,4 @@ """Fallback pure Python implementation of msgpack""" - from datetime import datetime as _DateTime import sys import struct @@ -148,6 +147,38 @@ def _unpack_from(f, b, o=0): else: _unpack_from = struct.unpack_from +_NO_FORMAT_USED = "" +_MSGPACK_HEADERS = { + 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), + 0xC5: (2, ">H", TYPE_BIN), + 0xC6: (4, ">I", TYPE_BIN), + 0xC7: (2, "Bb", TYPE_EXT), + 0xC8: (3, ">Hb", TYPE_EXT), + 0xC9: (5, ">Ib", TYPE_EXT), + 0xCA: (4, ">f"), + 0xCB: (8, ">d"), + 0xCC: (1, _NO_FORMAT_USED), + 0xCD: (2, ">H"), + 0xCE: (4, ">I"), + 0xCF: (8, ">Q"), + 0xD0: (1, "b"), + 0xD1: (2, ">h"), + 0xD2: (4, ">i"), + 0xD3: (8, ">q"), + 0xD4: (1, "b1s", TYPE_EXT), + 0xD5: (2, "b2s", TYPE_EXT), + 0xD6: (4, "b4s", TYPE_EXT), + 0xD7: (8, "b8s", TYPE_EXT), + 0xD8: (16, "b16s", TYPE_EXT), + 0xD9: (1, _NO_FORMAT_USED, TYPE_RAW), + 0xDA: (2, ">H", TYPE_RAW), + 0xDB: (4, ">I", TYPE_RAW), + 0xDC: (2, ">H", TYPE_ARRAY), + 0xDD: (4, ">I", TYPE_ARRAY), + 0xDE: (2, ">H", TYPE_MAP), + 0xDF: (4, ">I", TYPE_MAP), +} + class Unpacker(object): """Streaming unpacker. @@ -409,7 +440,7 @@ def _reserve(self, n, raise_outofdata=True): self._buff_i = 0 # rollback raise OutOfData - def _read_header(self, execute=EX_CONSTRUCT): + def _read_header(self): typ = TYPE_IMMEDIATE n = 0 obj = None @@ -442,187 +473,73 @@ def _read_header(self, execute=EX_CONSTRUCT): obj = False elif b == 0xC3: obj = True - elif b == 0xC4: - typ = TYPE_BIN - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xC5: - typ = TYPE_BIN - self._reserve(2) - n = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xC6: - typ = TYPE_BIN - self._reserve(4) - n = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 + elif 0xC4 <= b <= 0xC6: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + n = _unpack_from(fmt, self._buffer, self._buff_i)[0] + else: + n = self._buffer[self._buff_i] + self._buff_i += size if n > self._max_bin_len: raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) - elif b == 0xC7: # ext 8 - typ = TYPE_EXT - self._reserve(2) - L, n = _unpack_from("Bb", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xC7 <= b <= 0xC9: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + L, n = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) - elif b == 0xC8: # ext 16 - typ = TYPE_EXT - self._reserve(3) - L, n = _unpack_from(">Hb", self._buffer, self._buff_i) - self._buff_i += 3 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xC9: # ext 32 - typ = TYPE_EXT - self._reserve(5) - L, n = _unpack_from(">Ib", self._buffer, self._buff_i) - self._buff_i += 5 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xCA: - self._reserve(4) - obj = _unpack_from(">f", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xCB: - self._reserve(8) - obj = _unpack_from(">d", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xCC: - self._reserve(1) - obj = self._buffer[self._buff_i] - self._buff_i += 1 - elif b == 0xCD: - self._reserve(2) - obj = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xCE: - self._reserve(4) - obj = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xCF: - self._reserve(8) - obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xD0: - self._reserve(1) - obj = _unpack_from("b", self._buffer, self._buff_i)[0] - self._buff_i += 1 - elif b == 0xD1: - self._reserve(2) - obj = _unpack_from(">h", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xD2: - self._reserve(4) - obj = _unpack_from(">i", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xD3: - self._reserve(8) - obj = _unpack_from(">q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xD4: # fixext 1 - typ = TYPE_EXT - if self._max_ext_len < 1: - raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - self._reserve(2) - n, obj = _unpack_from("b1s", self._buffer, self._buff_i) - self._buff_i += 2 - elif b == 0xD5: # fixext 2 - typ = TYPE_EXT - if self._max_ext_len < 2: - raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - self._reserve(3) - n, obj = _unpack_from("b2s", self._buffer, self._buff_i) - self._buff_i += 3 - elif b == 0xD6: # fixext 4 - typ = TYPE_EXT - if self._max_ext_len < 4: - raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - self._reserve(5) - n, obj = _unpack_from("b4s", self._buffer, self._buff_i) - self._buff_i += 5 - elif b == 0xD7: # fixext 8 - typ = TYPE_EXT - if self._max_ext_len < 8: - raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - self._reserve(9) - n, obj = _unpack_from("b8s", self._buffer, self._buff_i) - self._buff_i += 9 - elif b == 0xD8: # fixext 16 - typ = TYPE_EXT - if self._max_ext_len < 16: - raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - self._reserve(17) - n, obj = _unpack_from("b16s", self._buffer, self._buff_i) - self._buff_i += 17 - elif b == 0xD9: - typ = TYPE_RAW - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xDA: - typ = TYPE_RAW - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xDB: - typ = TYPE_RAW - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 + elif 0xCA <= b <= 0xD3: + size, fmt = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] + else: + obj = self._buffer[self._buff_i] + self._buff_i += size + elif 0xD4 <= b <= 0xD8: + size, fmt, typ = _MSGPACK_HEADERS[b] + if self._max_ext_len < size: + raise ValueError( + "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) + ) + self._reserve(size + 1) + n, obj = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size + 1 + elif 0xD9 <= b <= 0xDB: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + else: + n = self._buffer[self._buff_i] + self._buff_i += size if n > self._max_str_len: raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) - elif b == 0xDC: - typ = TYPE_ARRAY - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xDD: - typ = TYPE_ARRAY - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 + elif 0xDC <= b <= 0xDD: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if n > self._max_array_len: raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xDE: - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xDE <= b <= 0xDF: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP - elif b == 0xDF: - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj def _unpack(self, execute=EX_CONSTRUCT): - typ, n, obj = self._read_header(execute) + typ, n, obj = self._read_header() if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: From 02e1f7623cd8d0fcd4763d542fc60e2957ee2046 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 01:11:32 +0000 Subject: [PATCH 015/105] build: Create tox environments using a known Cython version (#408) This change causes Tox to run the project's setup.py in a virtualenv (default path is .tox/.package). The required version of Cython is installed, rather than whatever version is installed system wide. --- pyproject.toml | 8 ++++++++ requirements.txt | 1 + tox.ini | 1 + 3 files changed, 10 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a9eb8aa9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + # Also declared in requirements.txt, if updating here please also update + # there + "Cython~=0.29.13", + "setuptools >= 35.0.2", +] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index a2cce258..180fe851 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +# Also declared in pyproject.toml, if updating here please also update there Cython~=0.29.13 diff --git a/tox.ini b/tox.ini index 607b182e..ace1ba90 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = {pypy,pypy3}-pure, py27-x86, py34-x86, +isolated_build = true [variants:pure] setenv= From cfae52437b0d146f74a14d69cb706712c1fa2c95 Mon Sep 17 00:00:00 2001 From: laike9m Date: Wed, 27 Jan 2021 15:33:14 -0800 Subject: [PATCH 016/105] Updated readme about Python 2 support (#456) --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index d8ce9baa..cb816485 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,6 @@ Python 2 and PyPy. But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy and Python 2. -Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, -Python 2 support will not be dropped in the foreseeable future. ### Windows From 1e728a2e0b7f263a4c77d6cdb6ec7c7e2a91872f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 12 Feb 2021 16:20:14 +0900 Subject: [PATCH 017/105] fix docstring (#459) --- msgpack/_unpacker.pyx | 91 ++++++++++++++++++++++++++++--------------- msgpack/fallback.py | 2 +- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e4f3f1e4..7bfc3af5 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -212,49 +212,76 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, cdef class Unpacker(object): - """ - MessagePack Packer + """Streaming unpacker. + + Arguments: - Usage:: + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - packer = Packer() - astream.write(packer.pack(a)) - astream.write(packer.pack(b)) + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) - Packer's constructor has some keyword arguments: + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) - :param callable default: - Convert user type to builtin type that Packer supports. - See also simplejson's document. + :param bool raw: + If true, unpack msgpack raw to Python bytes. + Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). - :param bool use_single_float: - Use single precision float type for float. (default: False) + :param int timestamp: + Control how timestamp type is unpacked: - :param bool autoreset: - Reset buffer after each pack and return its content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + 0 - Timestamp + 1 - float (Seconds from the EPOCH) + 2 - int (Nanoseconds from the EPOCH) + 3 - datetime.datetime (UTC). Python 2 is not supported. - :param bool use_bin_type: - Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. (default: True) + :param bool strict_map_key: + If true (default), only str or bytes are accepted for map (dict) keys. - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) - :param bool datetime: - If set to true, datetime with tzinfo is packed into Timestamp type. - Note that the tzinfo is stripped in the timestamp. - You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) :param str unicode_errors: - The error handler for encoding unicode. (default: 'strict') - DO NOT USE THIS!! This option is kept for very specific usage. + The error handler for decoding unicode. (default: 'strict') + This option should be used only when you have msgpack data which + contains invalid UTF-8 string. + + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means 2**32-1. + The default value is 100*1024*1024 (100MiB). + Raises `BufferFull` exception when it is insufficient. + You should set this parameter when unpacking data from untrusted source. + + :param int max_str_len: + Deprecated, use *max_buffer_size* instead. + Limits max length of str. (default: max_buffer_size) + + :param int max_bin_len: + Deprecated, use *max_buffer_size* instead. + Limits max length of bin. (default: max_buffer_size) + + :param int max_array_len: + Limits max length of array. + (default: max_buffer_size) + + :param int max_map_len: + Limits max length of map. + (default: max_buffer_size//2) + + :param int max_ext_len: + Deprecated, use *max_buffer_size* instead. + Limits max size of ext type. (default: max_buffer_size) Example of streaming deserialize from file-like object:: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 7dbc67ad..4bcc05d5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -260,7 +260,7 @@ class Unpacker(object): Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size) + unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: From 4b0819dca941ba3fd27ea127e835698c33705365 Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Tue, 16 Feb 2021 13:38:06 +0000 Subject: [PATCH 018/105] Remove redundant code (#460) --- test/test_unpack.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_unpack.py b/test/test_unpack.py index 057b7bf4..aa4c01f8 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -90,10 +90,3 @@ def test_unpacker_tell_read_bytes(): assert obj == unp assert pos == unpacker.tell() assert unpacker.read_bytes(n) == raw - - -if __name__ == "__main__": - test_unpack_array_header_from_file() - test_unpacker_hook_refcnt() - test_unpacker_ext_hook() - test_unpacker_tell() From 38357b928a2452d0889d80f0a2a721fa66f94c9a Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Fri, 26 Feb 2021 02:39:36 +0000 Subject: [PATCH 019/105] Fix error formatting (#463) --- msgpack/fallback.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 4bcc05d5..898fe146 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -455,18 +455,20 @@ def _read_header(self): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError( + "%s exceeds max_array_len(%s)" % (n, self._max_array_len) + ) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) elif b == 0xC0: obj = None elif b == 0xC2: @@ -518,7 +520,7 @@ def _read_header(self): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) obj = self._read(n) elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -526,14 +528,16 @@ def _read_header(self): (n,) = _unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError( + "%s exceeds max_array_len(%s)" % (n, self._max_array_len) + ) elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) (n,) = _unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj From 4ace82f1087ffa1ca3c44a27c7dd3338739efd0a Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Fri, 26 Feb 2021 12:08:06 +0000 Subject: [PATCH 020/105] Fix tox.ini (#465) There is no such thing as [variants] in the tox syntax. This resulted in MSGPACK_PUREPYTHON being unset in the "pure" test environments --- tox.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index ace1ba90..29c256d1 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,6 @@ envlist = py34-x86, isolated_build = true -[variants:pure] -setenv= - MSGPACK_PUREPYTHON=x - [testenv] deps= pytest @@ -20,6 +16,8 @@ commands= c,x86: python -c 'from msgpack import _cmsgpack' c,x86: py.test pure: py.test +setenv= + pure: MSGPACK_PUREPYTHON=x [testenv:py27-x86] basepython=python2.7-x86 From 44fd5777050c6583791609d3f77e05427bf878a3 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Sat, 27 Feb 2021 03:30:46 +0300 Subject: [PATCH 021/105] Remove unused PyObject_AsReadBuffer definition (#468) Also "old" buffer API was removed in Python 3.10 --- msgpack/_unpacker.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 7bfc3af5..9ecfdfbb 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -3,7 +3,6 @@ from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 object PyMemoryView_GetContiguous(object obj, int buffertype, char order) from libc.stdlib cimport * From 010de11bedde8d61ced5a382b44a8344d571c305 Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Sat, 27 Feb 2021 01:50:24 +0000 Subject: [PATCH 022/105] Make pure-python wheels and eggs possible (#467) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e88750a..751abff5 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def __init__(self, *args, **kwargs): macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] -if not PYPY and not PY2: +if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", From 38dba9634e4efa7886a777b9e7c739dc148da457 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 18 Mar 2021 14:35:54 -0700 Subject: [PATCH 023/105] cimport uint64_t instead of using ctypedef (#473) --- msgpack/_unpacker.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 9ecfdfbb..27facc0a 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -8,7 +8,7 @@ cdef extern from "Python.h": from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * -ctypedef unsigned long long uint64_t +from libc.stdint cimport uint64_t from .exceptions import ( BufferFull, From 09187421eb0ac7061de83c9c112d738aa1edd2cf Mon Sep 17 00:00:00 2001 From: Paul Melis Date: Tue, 16 Nov 2021 06:47:40 +0100 Subject: [PATCH 024/105] Improve exception message relating to strict_map_key (#485) --- msgpack/unpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 34212bcd..23aa6220 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -193,7 +193,7 @@ static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_un static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) { - PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name); + PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key when strict_map_key=True", Py_TYPE(k)->tp_name); return -1; } if (PyUnicode_CheckExact(k)) { From 9b84e490e7c78ac9bbd76dcf9ce71c1d0c978d81 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 14:53:08 +0900 Subject: [PATCH 025/105] Fix black formatting --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 898fe146..45408750 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -385,7 +385,7 @@ def feed(self, next_bytes): self._buffer.extend(view) def _consume(self): - """ Gets rid of the used parts of the buffer. """ + """Gets rid of the used parts of the buffer.""" self._stream_offset += self._buff_i - self._buf_checkpoint self._buf_checkpoint = self._buff_i From b3f72541925cda3b018942db45a59936184727e3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 16:19:47 +0900 Subject: [PATCH 026/105] Support Python 3.10 and Drop Python 3.5 (#487) * linux: Use manylinux2014 * mac: Drop Python 3.6 too --- .github/workflows/linux.yml | 11 +++++++++++ .github/workflows/mac.yml | 12 ++++++------ .github/workflows/windows.yaml | 14 ++++++++++++++ Makefile | 4 ++-- docker/buildwheel.sh | 4 +++- docker/shared.env | 2 +- setup.py | 4 +--- 7 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f9707b1c..811bc13d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -47,6 +47,17 @@ jobs: pytest -v test + - name: Set up Python 3.10 + uses: actions/setup-python@v1 + with: + python-version: "3.10" + + - name: Run test (3.10) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 78d944cc..4efe2cad 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -35,10 +35,10 @@ jobs: pytest -v test - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v1 with: - python-version: "3.9" + python-version: "3.10" - name: Build wheel run: | @@ -52,10 +52,10 @@ jobs: pytest -v test - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: "3.7" + python-version: "3.9" - name: Build wheel run: | @@ -69,10 +69,10 @@ jobs: pytest -v test - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: - python-version: "3.6" + python-version: "3.7" - name: Build wheel run: | diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 139a5a64..debe0747 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -77,6 +77,20 @@ jobs: run: | ci/runtests.sh + - name: Python 3.10 (amd64) + env: + PYTHON: "py -3.10-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.10 (x86) + env: + PYTHON: "py -3.10-32" + shell: bash + run: | + ci/runtests.sh + - name: Upload Wheels uses: actions/upload-artifact@v1 with: diff --git a/Makefile b/Makefile index 6f29aede..b50fa80c 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ update-docker: .PHONY: linux-wheel linux-wheel: - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_i686 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_x86_64 bash docker/buildwheel.sh .PHONY: linux-arm64-wheel linux-arm64-wheel: diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index 89a25706..ff34139d 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -7,10 +7,12 @@ set -e -x ARCH=`uname -p` echo "arch=$ARCH" +ls /opt/python + for V in "${PYTHON_VERSIONS[@]}"; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python - $PYBIN/python setup.py bdist_wheel + $PYBIN/python -m build -w done cd dist diff --git a/docker/shared.env b/docker/shared.env index 3601a075..80274ac6 100644 --- a/docker/shared.env +++ b/docker/shared.env @@ -1,7 +1,7 @@ PYTHON_VERSIONS=( + cp310-cp310 cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m - cp35-cp35m ) diff --git a/setup.py b/setup.py index 751abff5..01f125fb 100755 --- a/setup.py +++ b/setup.py @@ -125,14 +125,12 @@ def __init__(self, *args, **kwargs): }, license="Apache 2.0", classifiers=[ - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 8e358617e77d63a391f0c0f91b3e552214f2a49a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 17:42:42 +0900 Subject: [PATCH 027/105] mac: Provide Universal2 wheel (#488) * mac: Use cibuildwheel * Do not build wheel for PyPy. --- .github/workflows/mac.yml | 51 +++++++++++++++------------------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 4efe2cad..85844e7a 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,7 +1,6 @@ name: Build macOS Wheels on: push: - pull_request: create: jobs: @@ -12,10 +11,11 @@ jobs: - name: Checkout uses: actions/checkout@v1 - - name: Set up Python 3.8 + # Python 3.9 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: "3.8" + python-version: "3.9" - name: Cythonize run: | @@ -23,66 +23,53 @@ jobs: pip install -r requirements.txt make cython - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel + - name: Build wheels + uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_ARCHS_MACOS: x86_64 universal2 + CIBW_SKIP: pp* - name: Run test run: | + ls wheelhouse/ pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - + # Python 3.10 - name: Set up Python 3.10 uses: actions/setup-python@v1 with: python-version: "3.10" - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - - - name: Set up Python 3.9 + # Python 3.8 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: "3.9" - - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel + python-version: "3.8" - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - + # Python 3.7 - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: "3.7" - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test @@ -90,4 +77,4 @@ jobs: uses: actions/upload-artifact@v1 with: name: macos-wheels - path: ./dist/ + path: ./wheelhouse/ From cfa05d3fdc6290b4847e4781a06ac0668ea9dc18 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 17:47:16 +0900 Subject: [PATCH 028/105] Actions: Run CI only for PRs from forks. (#489) --- .github/workflows/black.yaml | 4 ++++ .github/workflows/linux.yml | 4 ++++ .github/workflows/windows.yaml | 7 ++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index eda8d076..c5459539 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -4,6 +4,10 @@ on: ["push", "pull_request"] jobs: black: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest steps: - name: Setup Python diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 811bc13d..95c672b2 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -6,6 +6,10 @@ on: jobs: build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index debe0747..0ce50f58 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -1,14 +1,15 @@ name: Build and test windows wheels on: push: - branches: - - master - - test pull_request: create: jobs: build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: windows-latest steps: - name: Checkout From e464cb44fa3af5ad3ecd83f9c045b16981d01bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= Date: Tue, 16 Nov 2021 09:49:47 +0100 Subject: [PATCH 029/105] Nicer error when packing a datetime without tzinfo (#466) --- msgpack/_packer.pyx | 2 ++ msgpack/fallback.py | 4 ++++ test/test_timestamp.py | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e6cd2c7d..396da0c2 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -285,6 +285,8 @@ cdef class Packer(object): o = self._default(o) default_used = 1 continue + elif self.datetime and PyDateTime_CheckExact(o): + PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) else: PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) return ret diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 45408750..b27acb29 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -874,6 +874,10 @@ def _pack( obj = self._default(obj) default_used = 1 continue + + if self._datetime and check(obj, _DateTime): + raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,)) + raise TypeError("Cannot serialize %r" % (obj,)) def pack(self, obj): diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 6a29be77..4e26489b 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -140,3 +140,19 @@ def test_issue451(): unpacked = msgpack.unpackb(packed, timestamp=3) assert dt == unpacked + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_pack_datetime_without_tzinfo(): + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) + with pytest.raises(ValueError, match="where tzinfo=None"): + packed = msgpack.packb(dt, datetime=True) + + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) + packed = msgpack.packb(dt, datetime=True, default=lambda x: None) + assert packed == msgpack.packb(None) + + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + packed = msgpack.packb(dt, datetime=True) + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == dt From 724e6200fd6b28b3562e48c73f7827a9c19dd11f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 17:52:01 +0900 Subject: [PATCH 030/105] 1.0.3rc1 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 1c83c8ed..3b462fa5 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 2) +version = (1, 0, 3, 'rc1') From e29b423de71cb6da323bce8742b1328603a7f9be Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 17 Nov 2021 11:03:06 +0900 Subject: [PATCH 031/105] black --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 3b462fa5..b7165245 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 3, 'rc1') +version = (1, 0, 3, "rc1") From 6129789e9f6ebccdb19b23c2cd1dc9551e57fc53 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 Nov 2021 16:18:17 +0900 Subject: [PATCH 032/105] Release v1.0.3 (#491) --- ChangeLog.rst | 9 +++++++++ msgpack/_version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 230cc301..fc6df680 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.3 +===== + +Release Date: 2021-11-24 JST + +* Fix Docstring (#459) +* Fix error formatting (#463) +* Improve error message about strict_map_key (#485) + 1.0.2 ===== diff --git a/msgpack/_version.py b/msgpack/_version.py index b7165245..fb878b35 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 3, "rc1") +version = (1, 0, 3) From bdf0511e29b02427437b03e7d8454bd5076e837f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 Nov 2021 14:43:55 +0900 Subject: [PATCH 033/105] Refactor CI (#492) * Use cibuildwheel to build wheels. * Use matrix --- .github/workflows/black.yaml | 2 +- .github/workflows/linux.yml | 103 --------------------------------- .github/workflows/mac.yml | 80 ------------------------- .github/workflows/test.yml | 45 ++++++++++++++ .github/workflows/wheel.yml | 51 ++++++++++++++++ .github/workflows/windows.yaml | 99 ------------------------------- .travis.yml | 89 ---------------------------- Makefile | 5 +- appveyor.yml | 50 ---------------- ci/runtests.bat | 9 --- ci/runtests.sh | 8 --- 11 files changed, 100 insertions(+), 441 deletions(-) delete mode 100644 .github/workflows/linux.yml delete mode 100644 .github/workflows/mac.yml create mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/wheel.yml delete mode 100644 .github/workflows/windows.yaml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml delete mode 100644 ci/runtests.bat delete mode 100644 ci/runtests.sh diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index c5459539..2961ed35 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '3.x' architecture: 'x64' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index 95c672b2..00000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Build Linux Wheels -on: - push: - pull_request: - create: - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Cythonize - shell: bash - run: | - pip install -U pip - pip -V - pip install -r requirements.txt - make cython - #python setup.py sdist - - - name: Build wheels - shell: bash - run: | - make linux-wheel - - - name: Install qemu-user-static for docker - shell: bash - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build arm64 wheels - shell: bash - run: | - make linux-arm64-wheel - - - name: Run test (3.8) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - - name: Set up Python 3.10 - uses: actions/setup-python@v1 - with: - python-version: "3.10" - - - name: Run test (3.10) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.9 - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - - name: Run test (3.9) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Run test (3.7) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - - name: Run test (3.6) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: linux-wheels - path: ./dist/wheelhouse/ diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml deleted file mode 100644 index 85844e7a..00000000 --- a/.github/workflows/mac.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build macOS Wheels -on: - push: - create: - -jobs: - build: - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v1 - - # Python 3.9 - - name: Set up Python 3.9 - uses: actions/setup-python@v1 - with: - python-version: "3.9" - - - name: Cythonize - run: | - pip install -U pip - pip install -r requirements.txt - make cython - - - name: Build wheels - uses: pypa/cibuildwheel@v2.2.2 - env: - CIBW_ARCHS_MACOS: x86_64 universal2 - CIBW_SKIP: pp* - - - name: Run test - run: | - ls wheelhouse/ - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.10 - - name: Set up Python 3.10 - uses: actions/setup-python@v1 - with: - python-version: "3.10" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.8 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: "3.8" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.7 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: "3.7" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: macos-wheels - path: ./wheelhouse/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..6e497e00 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,45 @@ +name: Run tests +on: + push: + branches: [main] + pull_request: + create: + +jobs: + test: + strategy: + matrix: + os: [ubuntu-20.04, windows-2022, macos-10.15] + py: ["3.10", "3.9", "3.8", "3.7", "3.6"] + + runs-on: ${{ matrix.os }} + name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py }} + cache: "pip" + + - name: Build + shell: bash + run: | + pip install -U pip + pip install -r requirements.txt pytest + make cython + pip install . + + - name: Test (C extension) + shell: bash + run: | + pytest -v test + + - name: Test (pure Python fallback) + shell: bash + run: | + MSGPACK_PUREPYTHON=1 pytest -v test + diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml new file mode 100644 index 00000000..5627b5fc --- /dev/null +++ b/.github/workflows/wheel.yml @@ -0,0 +1,51 @@ +name: Build Wheels +on: + push: + branches: [main] + create: + +jobs: + build_wheels: + strategy: + matrix: + os: [ubuntu-20.04, windows-2022, macos-10.15] + runs-on: ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + cache: "pip" + + - name: Prepare + shell: bash + run: | + pip install -r requirements.txt + make cython + + - name: Build + uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_TEST_REQUIRES: "pytest" + CIBW_TEST_COMMAND: "pytest {package}/test" + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_SKIP: pp* + + - name: Upload Wheels + uses: actions/upload-artifact@v1 + with: + name: Wheels + path: wheelhouse + diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml deleted file mode 100644 index 0ce50f58..00000000 --- a/.github/workflows/windows.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: Build and test windows wheels -on: - push: - pull_request: - create: - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Cythonize - shell: bash - run: | - pip install -U Cython - make cython - #python setup.py sdist - - - name: Python 3.6 (amd64) - env: - PYTHON: "py -3.6-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.6 (x86) - env: - PYTHON: "py -3.6-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.7 (amd64) - env: - PYTHON: "py -3.7-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.7 (x86) - env: - PYTHON: "py -3.7-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.8 (amd64) - env: - PYTHON: "py -3.8-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.8 (x86) - env: - PYTHON: "py -3.8-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.9 (amd64) - env: - PYTHON: "py -3.9-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.9 (x86) - env: - PYTHON: "py -3.9-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.10 (amd64) - env: - PYTHON: "py -3.10-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.10 (x86) - env: - PYTHON: "py -3.10-32" - shell: bash - run: | - ci/runtests.sh - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: win-wheels - path: ./dist diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4974d26c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,89 +0,0 @@ -version: ~> 1.0 -dist: xenial -language: python -cache: pip -arch: - - arm64 - -python: - # Available Python (PyPy) can be listed by: - # - # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - - "3.6" - - "3.7" - - "3.8" - - "3.9-dev" - -_pure: &pure - install: - - pip install -U pip - - pip install -U pytest pytest-cov codecov - - pip install . - script: - - pytest --cov=msgpack -v test - -matrix: - include: - - name: 32bit build - arch: amd64 - language: python - services: - - docker - env: - - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 - install: - - pip install -U pip - - pip install -r requirements.txt - - make cython - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - - arch: arm64 - name: arm64 32bit build - language: python - services: - - docker - env: - - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64 - install: - - pip install -U pip - - pip install -r requirements.txt - - make cython - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - - name: "Python 2 (fallback)" - python: "2.7" - <<: *pure - - - name: "pypy2.7" - arch: amd64 - python: "pypy2.7-7.1.1" - <<: *pure - - - name: "pypy3" - arch: amd64 - python: "pypy3.6-7.1.1" - <<: *pure - -install: - - pip install -U pip - - pip install -U pytest pytest-cov codecov - - pip install -r requirements.txt # Cython - - make cython - - pip install -e . - -script: - - python -c 'import sys; print(hex(sys.maxsize))' - - python -c 'from msgpack import _cmsgpack' - - pytest --cov=msgpack -v test - - MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test - -after_success: - - if [ -f .coverage ]; then - codecov; - fi - -# vim: sw=2 ts=2 diff --git a/Makefile b/Makefile index b50fa80c..02a4a170 100644 --- a/Makefile +++ b/Makefile @@ -31,8 +31,9 @@ clean: .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux1_i686 - docker pull quay.io/pypa/manylinux1_x86_64 + docker pull quay.io/pypa/manylinux2014_i686 + docker pull quay.io/pypa/manylinux2014_x86_64 + docker pull quay.io/pypa/manylinux2014_aarch64 .PHONY: linux-wheel linux-wheel: diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index f338e177..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -environment: - matrix: - # For Python versions available on Appveyor, see - # http://www.appveyor.com/docs/installed-software#python - - PYTHON: "C:\\Python36" - -install: - # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install -U pip" - - "%PYTHON%\\python.exe -m pip install -U cython" - - "%PYTHON%\\Scripts\\cython --cplus msgpack/_cmsgpack.pyx" - -build: off - -test_script: - # Put your test command here. - # Note that you must use the environment variable %PYTHON% to refer to - # the interpreter you're using - Appveyor does not do anything special - # to put the Python version you want to use on PATH. - - set PYTHON="C:\\Python27" - - ci\\runtests.bat - - set PYTHON="C:\\Python27-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python36" - - ci\\runtests.bat - - set PYTHON="C:\\Python36-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python37" - - ci\\runtests.bat - - set PYTHON="C:\\Python37-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python38" - - ci\\runtests.bat - - set PYTHON="C:\\Python38-x64" - - ci\\runtests.bat - -after_test: - # This step builds your wheels. - # Again, you need to use %PYTHON% to get the correct interpreter - -artifacts: - # bdist_wheel puts your built wheel in the dist directory - - path: dist\*.whl - -#on_success: -# You can use this step to upload your artifacts to a public website. -# See Appveyor's documentation for more details. Or you can simply -# access your wheels from the Appveyor "artifacts" tab for your build. - -# vim: set shiftwidth=2 diff --git a/ci/runtests.bat b/ci/runtests.bat deleted file mode 100644 index 4ae2f708..00000000 --- a/ci/runtests.bat +++ /dev/null @@ -1,9 +0,0 @@ -%PYTHON%\python.exe -m pip install -U pip wheel pytest -%PYTHON%\python.exe setup.py build_ext -i -%PYTHON%\python.exe setup.py install -%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))" -%PYTHON%\python.exe -c "from msgpack import _cmsgpack" -%PYTHON%\python.exe setup.py bdist_wheel -%PYTHON%\python.exe -m pytest -v test -SET EL=%ERRORLEVEL% -exit /b %EL% diff --git a/ci/runtests.sh b/ci/runtests.sh deleted file mode 100644 index 5d87f696..00000000 --- a/ci/runtests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -ex -${PYTHON} -VV -${PYTHON} -m pip install setuptools wheel pytest -${PYTHON} setup.py build_ext -if -${PYTHON} -c "from msgpack import _cmsgpack" -${PYTHON} setup.py bdist_wheel -${PYTHON} -m pytest -v test From 89ea57747ebcb0fad004a92ab00ebf13c10b2d51 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 19 Jan 2022 14:42:28 +0900 Subject: [PATCH 034/105] Don't define __*_ENDIAN__ macro on Unix. (#495) --- msgpack/fallback.py | 4 ---- setup.py | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b27acb29..9731a22f 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -11,7 +11,6 @@ def dict_iteritems(d): return d.iteritems() - else: int_types = int unicode = str @@ -32,7 +31,6 @@ def _is_recursionerror(e): and e.args[0].startswith("maximum recursion depth exceeded") ) - else: def _is_recursionerror(e): @@ -68,7 +66,6 @@ def write(self, s): def getvalue(self): return self.builder.build() - else: USING_STRINGBUILDER = False from io import BytesIO as StringIO @@ -143,7 +140,6 @@ def _unpack_from(f, b, o=0): """Explicit type cast for legacy struct.unpack_from""" return struct.unpack_from(f, bytes(b), o) - else: _unpack_from = struct.unpack_from diff --git a/setup.py b/setup.py index 01f125fb..502ed33e 100755 --- a/setup.py +++ b/setup.py @@ -79,12 +79,10 @@ def __init__(self, *args, **kwargs): libraries = [] +macros = [] + if sys.platform == "win32": libraries.append("ws2_32") - -if sys.byteorder == "big": - macros = [("__BIG_ENDIAN__", "1")] -else: macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] From cb50b2081b21e5cb4a364d292f55092c98aa1a6f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 3 Mar 2022 12:29:55 +0900 Subject: [PATCH 035/105] Update setuptools and black (#498) * Use setuptools * Use black==22.1.0 --- .github/workflows/black.yaml | 6 +++--- Makefile | 2 +- msgpack/__init__.py | 5 ++++- msgpack/_version.py | 1 - msgpack/ext.py | 8 +++---- msgpack/fallback.py | 12 +++++------ requirements.txt | 3 +++ setup.cfg | 32 +++++++++++++++++++++++++++ setup.py | 42 ++---------------------------------- test/test_limits.py | 12 +++++------ test/test_memoryview.py | 28 ++++++++++++------------ test/test_sequnpack.py | 4 ++-- test/test_timestamp.py | 12 +++++------ 13 files changed, 83 insertions(+), 84 deletions(-) delete mode 100644 msgpack/_version.py create mode 100644 setup.cfg diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 2961ed35..be137ae6 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -17,9 +17,9 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Black Code Formatter run: | - pip install black - black --diff --check msgpack/ test/ setup.py + pip install black==22.1.0 + black -S --diff --check msgpack/ test/ setup.py diff --git a/Makefile b/Makefile index 02a4a170..415dcfdd 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: black black: - black msgpack/ test/ setup.py + black -S msgpack/ test/ setup.py .PHONY: cython cython: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index d6705e22..1929df31 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,5 +1,4 @@ # coding: utf-8 -from ._version import version from .exceptions import * from .ext import ExtType, Timestamp @@ -7,6 +6,10 @@ import sys +version = (1, 0, 4, 'dev') +__version__ = "1.0.4dev" + + if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: from .fallback import Packer, unpackb, Unpacker else: diff --git a/msgpack/_version.py b/msgpack/_version.py deleted file mode 100644 index fb878b35..00000000 --- a/msgpack/_version.py +++ /dev/null @@ -1 +0,0 @@ -version = (1, 0, 3) diff --git a/msgpack/ext.py b/msgpack/ext.py index 4eb9dd65..25544c55 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -59,7 +59,7 @@ def __init__(self, seconds, nanoseconds=0): raise TypeError("seconds must be an interger") if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") - if not (0 <= nanoseconds < 10 ** 9): + if not (0 <= nanoseconds < 10**9): raise ValueError( "nanoseconds must be a non-negative integer less than 999999999." ) @@ -143,7 +143,7 @@ def from_unix(unix_sec): :type unix_float: int or float. """ seconds = int(unix_sec // 1) - nanoseconds = int((unix_sec % 1) * 10 ** 9) + nanoseconds = int((unix_sec % 1) * 10**9) return Timestamp(seconds, nanoseconds) def to_unix(self): @@ -161,7 +161,7 @@ def from_unix_nano(unix_ns): :param int unix_ns: Posix timestamp in nanoseconds. :rtype: Timestamp """ - return Timestamp(*divmod(unix_ns, 10 ** 9)) + return Timestamp(*divmod(unix_ns, 10**9)) def to_unix_nano(self): """Get the timestamp as a unixtime in nanoseconds. @@ -169,7 +169,7 @@ def to_unix_nano(self): :returns: posix timestamp in nanoseconds :rtype: int """ - return self.seconds * 10 ** 9 + self.nanoseconds + return self.seconds * 10**9 + self.nanoseconds def to_datetime(self): """Get the timestamp as a UTC datetime. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9731a22f..5f215e95 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -318,7 +318,7 @@ def __init__( self._buf_checkpoint = 0 if not max_buffer_size: - max_buffer_size = 2 ** 31 - 1 + max_buffer_size = 2**31 - 1 if max_str_len == -1: max_str_len = max_buffer_size if max_bin_len == -1: @@ -800,20 +800,20 @@ def _pack( raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) @@ -895,7 +895,7 @@ def pack_map_pairs(self, pairs): return ret def pack_array_header(self, n): - if n >= 2 ** 32: + if n >= 2**32: raise ValueError self._pack_array_header(n) if self._autoreset: @@ -904,7 +904,7 @@ def pack_array_header(self, n): return ret def pack_map_header(self, n): - if n >= 2 ** 32: + if n >= 2**32: raise ValueError self._pack_map_header(n) if self._autoreset: diff --git a/requirements.txt b/requirements.txt index 180fe851..9bdb4789 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there Cython~=0.29.13 + +# dev only tools. no need to add pyproject +black==22.1.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..1cb6ce36 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,32 @@ +[metadata] +name = msgpack +#version = attr: msgpack.__version__ +version = attr: msgpack.version +license = Apache 2.0 +author = Inada Naoki +author_email = songofacandy@gmail.com +description = MessagePack serializer +long_description = file: README.md +long_description_content_type = text/markdown +url = https://msgpack.org/ + +project_urls = + Documentation = https://msgpack-python.readthedocs.io/ + Source = https://github.com/msgpack/msgpack-python + Tracker = https://github.com/msgpack/msgpack-python/issues + +classifiers = + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + +[flake8] +max_line_length = 100 + diff --git a/setup.py b/setup.py index 502ed33e..9630cda0 100755 --- a/setup.py +++ b/setup.py @@ -4,10 +4,9 @@ import os import sys from glob import glob -from distutils.command.sdist import sdist from setuptools import setup, Extension - -from distutils.command.build_ext import build_ext +from setuptools.command.build_ext import build_ext +from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") @@ -65,12 +64,6 @@ def build_extension(self, ext): print(e) -exec(open("msgpack/_version.py").read()) - -version_str = ".".join(str(x) for x in version[:3]) -if len(version) > 3 and version[3] != "final": - version_str += version[3] - # Cython is required for sdist class Sdist(sdist): def __init__(self, *args, **kwargs): @@ -99,39 +92,8 @@ def __init__(self, *args, **kwargs): del libraries, macros -desc = "MessagePack (de)serializer." -with io.open("README.md", encoding="utf-8") as f: - long_desc = f.read() -del f - setup( - name="msgpack", - author="Inada Naoki", - author_email="songofacandy@gmail.com", - version=version_str, cmdclass={"build_ext": BuildExt, "sdist": Sdist}, ext_modules=ext_modules, packages=["msgpack"], - description=desc, - long_description=long_desc, - long_description_content_type="text/markdown", - url="https://msgpack.org/", - project_urls={ - "Documentation": "https://msgpack-python.readthedocs.io/", - "Source": "https://github.com/msgpack/msgpack-python", - "Tracker": "https://github.com/msgpack/msgpack-python/issues", - }, - license="Apache 2.0", - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - ], ) diff --git a/test/test_limits.py b/test/test_limits.py index 65e6bcc7..4314c2c0 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -16,12 +16,12 @@ def test_integer(): - x = -(2 ** 63) + x = -(2**63) assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): packb(x - 1) - x = 2 ** 64 - 1 + x = 2**64 - 1 assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): packb(x + 1) @@ -29,16 +29,16 @@ def test_integer(): def test_array_header(): packer = Packer() - packer.pack_array_header(2 ** 32 - 1) + packer.pack_array_header(2**32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2 ** 32) + packer.pack_array_header(2**32) def test_map_header(): packer = Packer() - packer.pack_map_header(2 ** 32 - 1) + packer.pack_map_header(2**32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2 ** 32) + packer.pack_array_header(2**32) def test_max_str_len(): diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 86b2c1f7..84941db8 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -53,46 +53,46 @@ def test_fixstr_from_float(): def test_str16_from_byte(): - _runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False) - _runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False) + _runtest("B", 2**8, b"\xda", b"\x01\x00", False) + _runtest("B", 2**16 - 1, b"\xda", b"\xff\xff", False) def test_str16_from_float(): - _runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False) - _runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False) + _runtest("f", 2**8, b"\xda", b"\x01\x00", False) + _runtest("f", 2**16 - 4, b"\xda", b"\xff\xfc", False) def test_str32_from_byte(): - _runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) + _runtest("B", 2**16, b"\xdb", b"\x00\x01\x00\x00", False) def test_str32_from_float(): - _runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) + _runtest("f", 2**16, b"\xdb", b"\x00\x01\x00\x00", False) def test_bin8_from_byte(): _runtest("B", 1, b"\xc4", b"\x01", True) - _runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True) + _runtest("B", 2**8 - 1, b"\xc4", b"\xff", True) def test_bin8_from_float(): _runtest("f", 4, b"\xc4", b"\x04", True) - _runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True) + _runtest("f", 2**8 - 4, b"\xc4", b"\xfc", True) def test_bin16_from_byte(): - _runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True) - _runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True) + _runtest("B", 2**8, b"\xc5", b"\x01\x00", True) + _runtest("B", 2**16 - 1, b"\xc5", b"\xff\xff", True) def test_bin16_from_float(): - _runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True) - _runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True) + _runtest("f", 2**8, b"\xc5", b"\x01\x00", True) + _runtest("f", 2**16 - 4, b"\xc5", b"\xff\xfc", True) def test_bin32_from_byte(): - _runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) + _runtest("B", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) def test_bin32_from_float(): - _runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) + _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6293a453..9f20c07e 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -118,8 +118,8 @@ def test_issue124(): def test_unpack_tell(): stream = io.BytesIO() - messages = [2 ** i - 1 for i in range(65)] - messages += [-(2 ** i) for i in range(1, 64)] + messages = [2**i - 1 for i in range(65)] + messages += [-(2**i) for i in range(1, 64)] messages += [ b"hello", b"hello" * 1000, diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 4e26489b..253228e7 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -10,31 +10,31 @@ def test_timestamp(): # timestamp32 - ts = Timestamp(2 ** 32 - 1) + ts = Timestamp(2**32 - 1) assert ts.to_bytes() == b"\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd6\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0 + assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0 # timestamp64 - ts = Timestamp(2 ** 34 - 1, 999999999) + ts = Timestamp(2**34 - 1, 999999999) assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd7\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999 # timestamp96 - ts = Timestamp(2 ** 63 - 1, 999999999) + ts = Timestamp(2**63 - 1, 999999999) assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xc7\x0c\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999 # negative fractional ts = Timestamp.from_unix(-2.3) # s: -3, ns: 700000000 From 849c8063817894f7dba166a19fbdbb9ffd8c2b80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2022 03:23:11 +0100 Subject: [PATCH 036/105] Use PyFloat_Pack8() on Python 3.11a7 (#499) Python 3.11a7 adds public functions: * PyFloat_Pack4(), PyFloat_Pack8() * PyFloat_Unpack4(), PyFloat_Unpack8() https://bugs.python.org/issue46906 --- msgpack/pack_template.h | 9 +++++++++ msgpack/unpack_template.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 0e940b84..7d479b6d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -568,7 +568,12 @@ static inline int msgpack_pack_float(msgpack_packer* x, float d) { unsigned char buf[5]; buf[0] = 0xca; + +#if PY_VERSION_HEX >= 0x030B00A7 + PyFloat_Pack4(d, (char *)&buf[1], 0); +#else _PyFloat_Pack4(d, &buf[1], 0); +#endif msgpack_pack_append_buffer(x, buf, 5); } @@ -576,7 +581,11 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d) { unsigned char buf[9]; buf[0] = 0xcb; +#if PY_VERSION_HEX >= 0x030B00A7 + PyFloat_Pack8(d, (char *)&buf[1], 0); +#else _PyFloat_Pack8(d, &buf[1], 0); +#endif msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 9924b9c6..8b9fcc19 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -243,10 +243,20 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize _msgpack_load32(uint32_t,n)+1, _ext_zero); case CS_FLOAT: { - double f = _PyFloat_Unpack4((unsigned char*)n, 0); + double f; +#if PY_VERSION_HEX >= 0x030B00A7 + f = PyFloat_Unpack4((const char*)n, 0); +#else + f = _PyFloat_Unpack4((unsigned char*)n, 0); +#endif push_fixed_value(_float, f); } case CS_DOUBLE: { - double f = _PyFloat_Unpack8((unsigned char*)n, 0); + double f; +#if PY_VERSION_HEX >= 0x030B00A7 + f = PyFloat_Unpack8((const char*)n, 0); +#else + f = _PyFloat_Unpack8((unsigned char*)n, 0); +#endif push_fixed_value(_double, f); } case CS_UINT_8: push_fixed_value(_uint8, *(uint8_t*)n); From 6a721faa778e4db23f76bc8a0f9adf85f59f69c7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 2 May 2022 02:26:53 -0600 Subject: [PATCH 037/105] Upgrade black (#505) --- .github/workflows/black.yaml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index be137ae6..0a0a737d 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -21,5 +21,5 @@ jobs: - name: Black Code Formatter run: | - pip install black==22.1.0 + pip install black==22.3.0 black -S --diff --check msgpack/ test/ setup.py diff --git a/requirements.txt b/requirements.txt index 9bdb4789..f557888d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ Cython~=0.29.13 # dev only tools. no need to add pyproject -black==22.1.0 +black==22.3.0 From b901b179d1976e09eb074803dcbb17d8d150c69f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 23 May 2022 04:52:09 +0000 Subject: [PATCH 038/105] Update Cython to 0.29.30 --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9eb8aa9..195795f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=0.29.13", + "Cython~=0.29.30", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index f557888d..9f3c1a0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there -Cython~=0.29.13 +Cython~=0.29.30 # dev only tools. no need to add pyproject black==22.3.0 From b75e3412fb8a2b6d6cd1da1b7063e14f6bfc0337 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 23 May 2022 05:01:08 +0000 Subject: [PATCH 039/105] Fix pip upgrade --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e497e00..2a019262 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Build shell: bash run: | - pip install -U pip + python -m pip install -U pip pip install -r requirements.txt pytest make cython pip install . From 500a238028bdebe123b502b07769578b5f0e8a3a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 19:46:51 +0900 Subject: [PATCH 040/105] Fix Unpacker max_buffer_length handling (#506) --- msgpack/_unpacker.pyx | 28 ++++++++++++---------------- msgpack/fallback.py | 2 ++ test/test_sequnpack.py | 11 ++++++++++- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 27facc0a..8b06661e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -440,34 +440,30 @@ cdef class Unpacker(object): self.buf_size = buf_size self.buf_tail = tail + _buf_len - cdef read_from_file(self): - next_bytes = self.file_like_read( - min(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) + cdef int read_from_file(self) except -1: + cdef Py_ssize_t remains = self.max_buffer_size - (self.buf_tail - self.buf_head) + if remains <= 0: + raise BufferFull + + next_bytes = self.file_like_read(min(self.read_size, remains)) if next_bytes: self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) else: self.file_like = None + return 0 cdef object _unpack(self, execute_fn execute, bint iter=0): cdef int ret cdef object obj cdef Py_ssize_t prev_head - if self.buf_head >= self.buf_tail and self.file_like is not None: - self.read_from_file() - while 1: prev_head = self.buf_head - if prev_head >= self.buf_tail: - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head + if prev_head < self.buf_tail: + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head + else: + ret = 0 if ret == 1: obj = unpack_data(&self.ctx) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5f215e95..f560c7b5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -423,6 +423,8 @@ def _reserve(self, n, raise_outofdata=True): # Read from file remain_bytes = -remain_bytes + if remain_bytes + len(self._buffer) > self._max_buffer_size: + raise BufferFull while remain_bytes > 0: to_read_bytes = max(self._read_size, remain_bytes) read_data = self.file_like.read(to_read_bytes) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 9f20c07e..c091076b 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -2,7 +2,7 @@ # coding: utf-8 import io from msgpack import Unpacker, BufferFull -from msgpack import pack +from msgpack import pack, packb from msgpack.exceptions import OutOfData from pytest import raises @@ -78,6 +78,15 @@ def test_maxbuffersize(): assert ord("b") == next(unpacker) +def test_maxbuffersize_file(): + buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20] * 2)) + unpacker = Unpacker(buff, read_size=1, max_buffer_size=19, max_bin_len=20) + assert unpacker.unpack() == b"a" * 10 + # assert unpacker.unpack() == [b"a" * 20]*2 + with raises(BufferFull): + print(unpacker.unpack()) + + def test_readbytes(): unpacker = Unpacker(read_size=3) unpacker.feed(b"foobar") From 63837a44d855017f1b2f667afa5ac684fd65591d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 20:13:07 +0900 Subject: [PATCH 041/105] ci: Update action versions. (#507) --- .github/workflows/test.yml | 7 +++---- .github/workflows/wheel.yml | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a019262..2a796aa2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,17 +10,17 @@ jobs: strategy: matrix: os: [ubuntu-20.04, windows-2022, macos-10.15] - py: ["3.10", "3.9", "3.8", "3.7", "3.6"] + py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.py }} cache: "pip" @@ -28,7 +28,6 @@ jobs: - name: Build shell: bash run: | - python -m pip install -U pip pip install -r requirements.txt pytest make cython pip install . diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 5627b5fc..541654db 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU if: runner.os == 'Linux' @@ -23,7 +23,7 @@ jobs: platforms: arm64 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 cache: "pip" @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.2.2 + uses: pypa/cibuildwheel@v2.5.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From a34dc945bfe39c1f2f91af2403d906069613ea41 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 May 2022 10:00:57 +0900 Subject: [PATCH 042/105] 1.0.4rc1 --- msgpack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 1929df31..81b2e67b 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import sys -version = (1, 0, 4, 'dev') -__version__ = "1.0.4dev" +version = (1, 0, 4, 'rc1') +__version__ = "1.0.4rc1" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From caadbf2df5a87039a52a5dcf4fc3f151bba70eed Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 May 2022 12:10:47 +0900 Subject: [PATCH 043/105] Use Actions to publish to PyPI --- .github/workflows/test.yml | 5 +++++ .github/workflows/wheel.yml | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a796aa2..d01d74cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,3 +42,8 @@ jobs: run: | MSGPACK_PUREPYTHON=1 pytest -v test + - name: Publish Wheels to TestPyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 541654db..9e8dce86 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -43,9 +43,15 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* - - name: Upload Wheels + - name: Upload Wheels to artifact uses: actions/upload-artifact@v1 with: name: Wheels path: wheelhouse + - name: Publish Wheels to TestPyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: wheelhouse + password: ${{ secrets.PYPI_API_TOKEN }} From b5acfd53833c3dbd379e539cc6e540cec83d0a99 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jun 2022 13:46:51 +0900 Subject: [PATCH 044/105] Release v1.0.4 (#509) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 10 ++++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 9e8dce86..d73898c2 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.5.0 + uses: pypa/cibuildwheel@v2.6.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index fc6df680..a11c8144 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.4 +===== + +Release Date: 2022-06-03 + +* Support Python 3.11 (beta). +* Don't define `__*_ENDIAN__` macro on Unix. by @methane in https://github.com/msgpack/msgpack-python/pull/495 +* Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in https://github.com/msgpack/msgpack-python/pull/499 +* Fix Unpacker max_buffer_length handling by @methane in https://github.com/msgpack/msgpack-python/pull/506 + 1.0.3 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 81b2e67b..50710218 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import sys -version = (1, 0, 4, 'rc1') -__version__ = "1.0.4rc1" +version = (1, 0, 4) +__version__ = "1.0.4" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From 9d45926a596028e39ec59dd909a56eb5e9e8fee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Tue, 2 Aug 2022 06:19:56 +0200 Subject: [PATCH 045/105] Usef `__BYTE_ORDER__` instead of `__BYTE_ORDER` (#513) __BYTE_ORDER__ is common predefined macro available on at least gcc and clang. __BYTE_ORDER is macro defined in platform specific headers. --- msgpack/sysdep.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index ed9c1bc0..ae28f0c5 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -61,14 +61,12 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif -#else -#include /* __BYTE_ORDER */ #endif #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define __BIG_ENDIAN__ #elif _WIN32 #define __LITTLE_ENDIAN__ From edca770071fc702e0b4c33f87fb0fa3682b486b4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 8 Aug 2022 15:08:40 +0900 Subject: [PATCH 046/105] Fix build error caused by ntohs, ntohl (#514) --- msgpack/sysdep.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index ae28f0c5..70673004 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -61,6 +61,8 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif +#else /* _WIN32 */ +#include /* ntohs, ntohl */ #endif #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) @@ -93,7 +95,7 @@ typedef unsigned int _msgpack_atomic_counter_t; #ifdef _WIN32 # if defined(ntohl) # define _msgpack_be32(x) ntohl(x) -# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400) +# elif defined(_byteswap_ulong) || defined(_MSC_VER) # define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x)) # else # define _msgpack_be32(x) \ @@ -106,7 +108,7 @@ typedef unsigned int _msgpack_atomic_counter_t; # define _msgpack_be32(x) ntohl(x) #endif -#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400) +#if defined(_byteswap_uint64) || defined(_MSC_VER) # define _msgpack_be64(x) (_byteswap_uint64(x)) #elif defined(bswap_64) # define _msgpack_be64(x) bswap_64(x) From 44a80603838ea480e66c9235036ff742f4013200 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 9 Sep 2022 09:16:12 +0200 Subject: [PATCH 047/105] Add python 3.11 wheels (#517) --- .github/workflows/wheel.yml | 4 ++-- setup.cfg | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d73898c2..770b565a 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: build_wheels: strategy: matrix: - os: [ubuntu-20.04, windows-2022, macos-10.15] + os: [ubuntu-22.04, windows-2022, macos-10.15] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.6.0 + uses: pypa/cibuildwheel@v2.9.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/setup.cfg b/setup.cfg index 1cb6ce36..e3782842 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Intended Audience :: Developers From c3995669f1f821596714240c2cd07943810f8658 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 08:08:58 +0000 Subject: [PATCH 048/105] Remove unused code --- msgpack/pack.h | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 4f3ce1d9..1e849acc 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -71,7 +71,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ static inline int msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) { -#if PY_MAJOR_VERSION >= 3 assert(PyUnicode_Check(o)); Py_ssize_t len; @@ -87,31 +86,6 @@ msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) if (ret) return ret; return msgpack_pack_raw_body(pk, buf, len); -#else - PyObject *bytes; - Py_ssize_t len; - int ret; - - // py2 - bytes = PyUnicode_AsUTF8String(o); - if (bytes == NULL) - return -1; - - len = PyString_GET_SIZE(bytes); - if (len > limit) { - Py_DECREF(bytes); - return -2; - } - - ret = msgpack_pack_raw(pk, len); - if (ret) { - Py_DECREF(bytes); - return -1; - } - ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len); - Py_DECREF(bytes); - return ret; -#endif } #ifdef __cplusplus From b82d0b62f187552b8108602d7b0451ac362a29cc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 19:13:44 +0900 Subject: [PATCH 049/105] fallback: Fix packing multidim memoryview (#527) Fix #526 --- msgpack/fallback.py | 2 +- test/test_memoryview.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f560c7b5..e8cebc1b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -814,7 +814,7 @@ def _pack( self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): - n = len(obj) * obj.itemsize + n = obj.nbytes if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 84941db8..a0939a69 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -96,3 +96,11 @@ def test_bin32_from_byte(): def test_bin32_from_float(): _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) + + +def test_multidim_memoryview(): + # See https://github.com/msgpack/msgpack-python/issues/526 + view = memoryview(b"\00" * 6) + data = view.cast(view.format, (3, 2)) + packed = packb(data) + assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00' From 10082295536098d90681da5d7199ca384e8b8ff8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 19:47:15 +0900 Subject: [PATCH 050/105] Release v1.0.5rc1 (#528) --- .github/workflows/black.yaml | 4 ++-- .github/workflows/test.yml | 6 +++--- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 9 +++++++++ msgpack/__init__.py | 4 ++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 0a0a737d..1e28b7b5 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.x' architecture: 'x64' - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Black Code Formatter run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d01d74cb..5e41167f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,8 @@ jobs: test: strategy: matrix: - os: [ubuntu-20.04, windows-2022, macos-10.15] - py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"] + os: [ubuntu-22.04, windows-2022, macos-10.15] + py: ["3.11", "3.10", "3.9", "3.8", "3.7"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} cache: "pip" diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 770b565a..b2879fe5 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.9.0 + uses: pypa/cibuildwheel@v2.12.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index a11c8144..1b7ae2c3 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.5rc1 +======== + +Release Date: 2023-01-18 + +* Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) +* Add Python 3.11 wheels (#517) +* fallback: Fix packing multidimensional memoryview (#527) + 1.0.4 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 50710218..501b291b 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import sys -version = (1, 0, 4) -__version__ = "1.0.4" +version = (1, 0, 5, 'rc', 1) +__version__ = "1.0.5rc1" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From e3ef909c47e5a245fc9d711e9d974a5f9df99303 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 13:07:24 +0000 Subject: [PATCH 051/105] Action: Use setup-python@v4 --- .github/workflows/wheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index b2879fe5..23050088 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -22,10 +22,10 @@ jobs: with: platforms: arm64 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 + - name: Set up Python 3.x + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.x" cache: "pip" - name: Prepare From dcb775031c0b1d575b90e822e81e845ebfda4a2e Mon Sep 17 00:00:00 2001 From: Anthon van der Neut Date: Sun, 5 Mar 2023 15:45:38 +0100 Subject: [PATCH 052/105] minor type in exception message (#533) interger -> integer --- msgpack/ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 25544c55..23e0d6b4 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -56,7 +56,7 @@ def __init__(self, seconds, nanoseconds=0): Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ if not isinstance(seconds, int_types): - raise TypeError("seconds must be an interger") + raise TypeError("seconds must be an integer") if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): From aa9ce3e2bbc1d3d0476396892c46e704292455ab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jun 2022 14:37:21 +0900 Subject: [PATCH 053/105] Action: Run publish on tag creation. --- .github/workflows/wheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 23050088..5f103a37 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -49,8 +49,8 @@ jobs: name: Wheels path: wheelhouse - - name: Publish Wheels to TestPyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + - name: Publish Wheels to PyPI + if: github.event_name == 'create' && github.event.ref_type == 'tag' uses: pypa/gh-action-pypi-publish@release/v1 with: packages_dir: wheelhouse From 4c55f809fe2231130cf99b20538b26b92f1bea31 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 00:43:28 +0900 Subject: [PATCH 054/105] Release v1.0.5 (#534) --- ChangeLog.rst | 6 +++--- msgpack/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 1b7ae2c3..53547996 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ -1.0.5rc1 -======== +1.0.5 +===== -Release Date: 2023-01-18 +Release Date: 2023-03-08 * Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) * Add Python 3.11 wheels (#517) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 501b291b..1300b866 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import sys -version = (1, 0, 5, 'rc', 1) -__version__ = "1.0.5rc1" +version = (1, 0, 5) +__version__ = "1.0.5" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From 35b2d246cfdb19484caa5789512bf71ee378caec Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 00:47:52 +0900 Subject: [PATCH 055/105] Action: Update wheel workflow --- .github/workflows/wheel.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 5f103a37..6cf2fe94 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: build_wheels: strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-10.15] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -48,10 +48,3 @@ jobs: with: name: Wheels path: wheelhouse - - - name: Publish Wheels to PyPI - if: github.event_name == 'create' && github.event.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages_dir: wheelhouse - password: ${{ secrets.PYPI_API_TOKEN }} From 0516c2c2a97ef48a2becf30bc8b2365ca16199f1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 01:22:38 +0900 Subject: [PATCH 056/105] Action: Update test workflow --- .github/workflows/test.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e41167f..88781025 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: test: strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-10.15] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] py: ["3.11", "3.10", "3.9", "3.8", "3.7"] runs-on: ${{ matrix.os }} @@ -41,9 +41,3 @@ jobs: shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test - - - name: Publish Wheels to TestPyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} From 802cbc9495ed059b62b46f057cf3d71f756e2480 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 1 Apr 2023 00:02:25 +0900 Subject: [PATCH 057/105] Add security policy --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..75f0c541 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. \ No newline at end of file From 45f848695c855966d8a46656d8de1a2734d934ae Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Sat, 8 Apr 2023 02:18:25 -0300 Subject: [PATCH 058/105] fix: build status badge (#538) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb816485..7f7c4234 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MessagePack for Python -[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python) +[![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml) [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) ## What's this From feec06206c7f8dd8efeac6177badbf5b256b36e4 Mon Sep 17 00:00:00 2001 From: sblondon Date: Sun, 21 May 2023 09:26:39 +0200 Subject: [PATCH 059/105] Drop python2 support (#519) The PR removes python2 references and cases. Close #518 Co-authored-by: Inada Naoki --- README.md | 4 +-- docs/conf.py | 20 ++++++------ msgpack/__init__.py | 2 +- msgpack/_packer.pyx | 1 - msgpack/_unpacker.pyx | 2 +- msgpack/ext.py | 24 +++----------- msgpack/fallback.py | 70 ++++++++++++----------------------------- setup.py | 3 +- test/test_buffer.py | 1 - test/test_case.py | 2 +- test/test_except.py | 2 +- test/test_extension.py | 15 ++------- test/test_memoryview.py | 5 --- test/test_pack.py | 6 ---- test/test_timestamp.py | 27 +++++++--------- test/test_unpack.py | 4 +-- tox.ini | 13 -------- 17 files changed, 58 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index 7f7c4234..61f99e1f 100644 --- a/README.md +++ b/README.md @@ -220,9 +220,9 @@ and `raw=True` options. ```pycon >>> import msgpack ->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) +>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=False), raw=True) [b'spam', b'eggs'] ->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) +>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=True), raw=False) [b'spam', 'eggs'] ``` diff --git a/docs/conf.py b/docs/conf.py index 6b432be0..91ce77f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,8 +40,8 @@ master_doc = "index" # General information about the project. -project = u"msgpack" -copyright = u"Inada Naoki" +project = "msgpack" +copyright = "Inada Naoki" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -181,7 +181,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"), + ("index", "msgpack.tex", "msgpack Documentation", "Author", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -209,7 +209,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)] +man_pages = [("index", "msgpack", "msgpack Documentation", ["Author"], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -224,8 +224,8 @@ ( "index", "msgpack", - u"msgpack Documentation", - u"Author", + "msgpack Documentation", + "Author", "msgpack", "One line description of project.", "Miscellaneous", @@ -245,10 +245,10 @@ # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u"msgpack" -epub_author = u"Author" -epub_publisher = u"Author" -epub_copyright = u"2013, Author" +epub_title = "msgpack" +epub_author = "Author" +epub_publisher = "Author" +epub_copyright = "2013, Author" # The language of the text. It defaults to the language option # or en if the language is not set. diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 1300b866..6c10dc23 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -10,7 +10,7 @@ __version__ = "1.0.5" -if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: +if os.environ.get("MSGPACK_PUREPYTHON"): from .fallback import Packer, unpackb, Unpacker else: try: diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 396da0c2..074b39fc 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -98,7 +98,6 @@ cdef class Packer(object): If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 8b06661e..d5dc5ea5 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -236,7 +236,7 @@ cdef class Unpacker(object): 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) - 3 - datetime.datetime (UTC). Python 2 is not supported. + 3 - datetime.datetime (UTC). :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. diff --git a/msgpack/ext.py b/msgpack/ext.py index 23e0d6b4..07f96a58 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -5,19 +5,6 @@ import struct -PY2 = sys.version_info[0] == 2 - -if PY2: - int_types = (int, long) - _utc = None -else: - int_types = int - try: - _utc = datetime.timezone.utc - except AttributeError: - _utc = datetime.timezone(datetime.timedelta(0)) - - class ExtType(namedtuple("ExtType", "code data")): """ExtType represents ext type in msgpack.""" @@ -55,9 +42,9 @@ def __init__(self, seconds, nanoseconds=0): Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ - if not isinstance(seconds, int_types): + if not isinstance(seconds, int): raise TypeError("seconds must be an integer") - if not isinstance(nanoseconds, int_types): + if not isinstance(nanoseconds, int): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): raise ValueError( @@ -174,11 +161,10 @@ def to_unix_nano(self): def to_datetime(self): """Get the timestamp as a UTC datetime. - Python 2 is not supported. - :rtype: datetime. """ - return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta( + utc = datetime.timezone.utc + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( seconds=self.to_unix() ) @@ -186,8 +172,6 @@ def to_datetime(self): def from_datetime(dt): """Create a Timestamp from datetime with tzinfo. - Python 2 is not supported. - :rtype: Timestamp """ return Timestamp.from_unix(dt.timestamp()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index e8cebc1b..618c3622 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,22 +4,6 @@ import struct -PY2 = sys.version_info[0] == 2 -if PY2: - int_types = (int, long) - - def dict_iteritems(d): - return d.iteritems() - -else: - int_types = int - unicode = str - xrange = range - - def dict_iteritems(d): - return d.items() - - if sys.version_info < (3, 5): # Ugly hack... RecursionError = RuntimeError @@ -134,15 +118,6 @@ def unpackb(packed, **kwargs): return ret -if sys.version_info < (2, 7, 6): - - def _unpack_from(f, b, o=0): - """Explicit type cast for legacy struct.unpack_from""" - return struct.unpack_from(f, bytes(b), o) - -else: - _unpack_from = struct.unpack_from - _NO_FORMAT_USED = "" _MSGPACK_HEADERS = { 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), @@ -202,7 +177,7 @@ class Unpacker(object): 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) - 3 - datetime.datetime (UTC). Python 2 is not supported. + 3 - datetime.datetime (UTC). :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. @@ -477,7 +452,7 @@ def _read_header(self): size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - n = _unpack_from(fmt, self._buffer, self._buff_i)[0] + n = struct.unpack_from(fmt, self._buffer, self._buff_i)[0] else: n = self._buffer[self._buff_i] self._buff_i += size @@ -487,7 +462,7 @@ def _read_header(self): elif 0xC7 <= b <= 0xC9: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - L, n = _unpack_from(fmt, self._buffer, self._buff_i) + L, n = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -496,7 +471,7 @@ def _read_header(self): size, fmt = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] + obj = struct.unpack_from(fmt, self._buffer, self._buff_i)[0] else: obj = self._buffer[self._buff_i] self._buff_i += size @@ -507,13 +482,13 @@ def _read_header(self): "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) ) self._reserve(size + 1) - n, obj = _unpack_from(fmt, self._buffer, self._buff_i) + n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size + 1 elif 0xD9 <= b <= 0xDB: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) else: n = self._buffer[self._buff_i] self._buff_i += size @@ -523,7 +498,7 @@ def _read_header(self): elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: raise ValueError( @@ -532,7 +507,7 @@ def _read_header(self): elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) @@ -554,12 +529,12 @@ def _unpack(self, execute=EX_CONSTRUCT): # TODO should we eliminate the recursion? if typ == TYPE_ARRAY: if execute == EX_SKIP: - for i in xrange(n): + for i in range(n): # TODO check whether we need to call `list_hook` self._unpack(EX_SKIP) return ret = newlist_hint(n) - for i in xrange(n): + for i in range(n): ret.append(self._unpack(EX_CONSTRUCT)) if self._list_hook is not None: ret = self._list_hook(ret) @@ -567,7 +542,7 @@ def _unpack(self, execute=EX_CONSTRUCT): return ret if self._use_list else tuple(ret) if typ == TYPE_MAP: if execute == EX_SKIP: - for i in xrange(n): + for i in range(n): # TODO check whether we need to call hooks self._unpack(EX_SKIP) self._unpack(EX_SKIP) @@ -575,17 +550,17 @@ def _unpack(self, execute=EX_CONSTRUCT): if self._object_pairs_hook is not None: ret = self._object_pairs_hook( (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) - for _ in xrange(n) + for _ in range(n) ) else: ret = {} - for _ in xrange(n): + for _ in range(n): key = self._unpack(EX_CONSTRUCT) - if self._strict_map_key and type(key) not in (unicode, bytes): + if self._strict_map_key and type(key) not in (str, bytes): raise ValueError( "%s is not allowed for map key" % str(type(key)) ) - if not PY2 and type(key) is str: + if type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: @@ -698,7 +673,6 @@ class Packer(object): If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') @@ -743,8 +717,6 @@ def __init__( self._autoreset = autoreset self._use_bin_type = use_bin_type self._buffer = StringIO() - if PY2 and datetime: - raise ValueError("datetime is not supported in Python 2") self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" if default is not None: @@ -774,7 +746,7 @@ def _pack( if obj: return self._buffer.write(b"\xc3") return self._buffer.write(b"\xc2") - if check(obj, int_types): + if check(obj, int): if 0 <= obj < 0x80: return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: @@ -806,7 +778,7 @@ def _pack( raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) - if check(obj, unicode): + if check(obj, str): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) if n >= 2**32: @@ -855,13 +827,11 @@ def _pack( if check(obj, list_types): n = len(obj) self._pack_array_header(n) - for i in xrange(n): + for i in range(n): self._pack(obj[i], nest_limit - 1) return if check(obj, dict): - return self._pack_map_pairs( - len(obj), dict_iteritems(obj), nest_limit - 1 - ) + return self._pack_map_pairs(len(obj), obj.items(), nest_limit - 1) if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None: obj = Timestamp.from_datetime(obj) @@ -1004,7 +974,7 @@ def reset(self): def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER or PY2: + if USING_STRINGBUILDER: return memoryview(self.bytes()) else: return self._buffer.getbuffer() diff --git a/setup.py b/setup.py index 9630cda0..15ba7741 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ PYPY = hasattr(sys, "pypy_version_info") -PY2 = sys.version_info[0] == 2 class NoCython(Exception): @@ -79,7 +78,7 @@ def __init__(self, *args, **kwargs): macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] -if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): +if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", diff --git a/test/test_buffer.py b/test/test_buffer.py index 62507cf4..04470584 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -6,7 +6,6 @@ from msgpack import packb, unpackb -@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported") def test_unpack_buffer(): from array import array diff --git a/test/test_case.py b/test/test_case.py index a0a3c5ad..1c4e3229 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -134,4 +134,4 @@ def test_match(): def test_unicode(): - assert unpackb(packb(u"foobar"), use_list=1) == u"foobar" + assert unpackb(packb("foobar"), use_list=1) == "foobar" diff --git a/test/test_except.py b/test/test_except.py index 5544f2bc..745ebecb 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -53,7 +53,7 @@ def test_invalidvalue(): def test_strict_map_key(): - valid = {u"unicode": 1, b"bytes": 2} + valid = {"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) assert valid == unpackb(packed, raw=False, strict_map_key=True) diff --git a/test/test_extension.py b/test/test_extension.py index 6b365751..dfbe4350 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -55,10 +55,7 @@ def ext_hook(code, data): print("ext_hook called", code, data) assert code == 123 obj = array.array("d") - try: - obj.frombytes(data) - except AttributeError: # PY2 - obj.fromstring(data) + obj.frombytes(data) return obj obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])] @@ -67,20 +64,14 @@ def ext_hook(code, data): assert obj == obj2 -import sys - -if sys.version > "3": - long = int - - def test_overriding_hooks(): def default(obj): - if isinstance(obj, long): + if isinstance(obj, int): return {"__type__": "long", "__data__": str(obj)} else: return obj - obj = {"testval": long(1823746192837461928374619)} + obj = {"testval": 1823746192837461928374619} refobj = {"testval": default(obj["testval"])} refout = msgpack.packb(refobj) assert isinstance(refout, (str, bytes)) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index a0939a69..63beab10 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -7,11 +7,6 @@ import sys -pytestmark = pytest.mark.skipif( - sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol" -) - - def make_array(f, data): a = array(f) a.frombytes(data) diff --git a/test/test_pack.py b/test/test_pack.py index a51d84c9..65c9cb17 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -80,9 +80,6 @@ def testPackByteArrays(): check(td) -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" -) def testIgnoreUnicodeErrors(): re = unpackb( packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" @@ -96,9 +93,6 @@ def testStrictUnicodeUnpack(): unpackb(packed, raw=False, use_list=1) -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" -) def testIgnoreErrorsPack(): re = unpackb( packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 253228e7..af84a2ff 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -4,9 +4,6 @@ import msgpack from msgpack.ext import Timestamp -if sys.version_info[0] > 2: - from msgpack.ext import _utc - def test_timestamp(): # timestamp32 @@ -85,33 +82,33 @@ def test_timestamp_to(): assert t.to_unix_nano() == 42000014000 -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_timestamp_datetime(): t = Timestamp(42, 14) - assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + utc = datetime.timezone.utc + assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_unpack_datetime(): t = Timestamp(42, 14) + utc = datetime.timezone.utc packed = msgpack.packb(t) unpacked = msgpack.unpackb(packed, timestamp=3) - assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_unpack_before_epoch(): - t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc) + utc = datetime.timezone.utc + t_in = datetime.datetime(1960, 1, 1, tzinfo=utc) packed = msgpack.packb(t_in, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == t_in -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime(): t = Timestamp(42, 14000) dt = t.to_datetime() - assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + utc = datetime.timezone.utc + assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) packed2 = msgpack.packb(t) @@ -131,10 +128,10 @@ def test_pack_datetime(): assert msgpack.unpackb(packed) is None -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_issue451(): # https://github.com/msgpack/msgpack-python/issues/451 - dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc) + utc = datetime.timezone.utc + dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) assert packed == b"\xd6\xff\xf4\x86eL" @@ -142,7 +139,6 @@ def test_issue451(): assert dt == unpacked -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime_without_tzinfo(): dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) with pytest.raises(ValueError, match="where tzinfo=None"): @@ -152,7 +148,8 @@ def test_pack_datetime_without_tzinfo(): packed = msgpack.packb(dt, datetime=True, default=lambda x: None) assert packed == msgpack.packb(None) - dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + utc = datetime.timezone.utc + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == dt diff --git a/test/test_unpack.py b/test/test_unpack.py index aa4c01f8..c7141024 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -70,7 +70,7 @@ def _hook(self, code, data): def test_unpacker_tell(): - objects = 1, 2, u"abc", u"def", u"ghi" + objects = 1, 2, "abc", "def", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" positions = 1, 2, 6, 10, 14 unpacker = Unpacker(BytesIO(packed)) @@ -80,7 +80,7 @@ def test_unpacker_tell(): def test_unpacker_tell_read_bytes(): - objects = 1, u"abc", u"ghi" + objects = 1, "abc", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" raw_data = b"\x02", b"\xa3def", b"" lenghts = 1, 4, 999 diff --git a/tox.ini b/tox.ini index 29c256d1..1ef2d189 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,7 @@ [tox] envlist = - py27-pure, {py35,py36,py37,py38}-{c,pure}, {pypy,pypy3}-pure, - py27-x86, py34-x86, isolated_build = true @@ -19,17 +17,6 @@ commands= setenv= pure: MSGPACK_PUREPYTHON=x -[testenv:py27-x86] -basepython=python2.7-x86 -deps= - pytest - -changedir=test -commands= - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - py.test - [testenv:py34-x86] basepython=python3.4-x86 deps= From c8d0751fe3375a5e8005b5edf955cd5904aaec2f Mon Sep 17 00:00:00 2001 From: Evgeny Markov Date: Tue, 23 May 2023 18:41:08 +0200 Subject: [PATCH 060/105] Drop Python 3.6 support (#543) The following steps have been taken: 1. Black was updated to latest version. The code has been formatted with the new version. 2. The pyupgrade utility is installed. This helped to remove all the code that was needed to support Python < 3.7. Fix #541. Co-authored-by: Inada Naoki --- Makefile | 8 ++++- msgpack/__init__.py | 1 - msgpack/ext.py | 21 ++++--------- msgpack/fallback.py | 70 ++++++++++++----------------------------- pyproject.toml | 5 +++ requirements.txt | 8 +++-- setup.cfg | 1 - setup.py | 9 ++---- test/test_buffer.py | 1 - test/test_case.py | 7 ++--- test/test_except.py | 1 - test/test_extension.py | 7 ++--- test/test_format.py | 5 +-- test/test_limits.py | 2 -- test/test_memoryview.py | 1 - test/test_newspec.py | 2 -- test/test_obj.py | 1 - test/test_pack.py | 12 +++---- test/test_seq.py | 1 - test/test_sequnpack.py | 1 - test/test_stricttype.py | 12 +++---- test/test_subtype.py | 1 - test/test_unpack.py | 2 +- 23 files changed, 60 insertions(+), 119 deletions(-) diff --git a/Makefile b/Makefile index 415dcfdd..e4f22da4 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,16 @@ +PYTHON_SOURCES = msgpack test setup.py + .PHONY: all all: cython python setup.py build_ext -i -f .PHONY: black black: - black -S msgpack/ test/ setup.py + black $(PYTHON_SOURCES) + +.PHONY: pyupgrade +pyupgrade: + @find $(PYTHON_SOURCES) -name '*.py' -type f -exec pyupgrade --py37-plus '{}' \; .PHONY: cython cython: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 6c10dc23..638236fe 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,4 +1,3 @@ -# coding: utf-8 from .exceptions import * from .ext import ExtType, Timestamp diff --git a/msgpack/ext.py b/msgpack/ext.py index 07f96a58..97942949 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,4 +1,3 @@ -# coding: utf-8 from collections import namedtuple import datetime import sys @@ -15,10 +14,10 @@ def __new__(cls, code, data): raise TypeError("data must be bytes") if not 0 <= code <= 127: raise ValueError("code must be 0~127") - return super(ExtType, cls).__new__(cls, code, data) + return super().__new__(cls, code, data) -class Timestamp(object): +class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python @@ -47,24 +46,18 @@ def __init__(self, seconds, nanoseconds=0): if not isinstance(nanoseconds, int): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): - raise ValueError( - "nanoseconds must be a non-negative integer less than 999999999." - ) + raise ValueError("nanoseconds must be a non-negative integer less than 999999999.") self.seconds = seconds self.nanoseconds = nanoseconds def __repr__(self): """String representation of Timestamp.""" - return "Timestamp(seconds={0}, nanoseconds={1})".format( - self.seconds, self.nanoseconds - ) + return f"Timestamp(seconds={self.seconds}, nanoseconds={self.nanoseconds})" def __eq__(self, other): """Check for equality with another Timestamp object""" if type(other) is self.__class__: - return ( - self.seconds == other.seconds and self.nanoseconds == other.nanoseconds - ) + return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds return False def __ne__(self, other): @@ -164,9 +157,7 @@ def to_datetime(self): :rtype: datetime. """ utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( - seconds=self.to_unix() - ) + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) @staticmethod def from_datetime(dt): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 618c3622..ac1eaf4c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,23 +4,6 @@ import struct -if sys.version_info < (3, 5): - # Ugly hack... - RecursionError = RuntimeError - - def _is_recursionerror(e): - return ( - len(e.args) == 1 - and isinstance(e.args[0], str) - and e.args[0].startswith("maximum recursion depth exceeded") - ) - -else: - - def _is_recursionerror(e): - return True - - if hasattr(sys, "pypy_version_info"): # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. @@ -32,7 +15,7 @@ def _is_recursionerror(e): from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True - class StringIO(object): + class StringIO: def __init__(self, s=b""): if s: self.builder = StringBuilder(len(s)) @@ -109,10 +92,8 @@ def unpackb(packed, **kwargs): ret = unpacker._unpack() except OutOfData: raise ValueError("Unpack failed: incomplete input") - except RecursionError as e: - if _is_recursionerror(e): - raise StackError - raise + except RecursionError: + raise StackError if unpacker._got_extradata(): raise ExtraData(ret, unpacker._get_extradata()) return ret @@ -151,7 +132,7 @@ def unpackb(packed, **kwargs): } -class Unpacker(object): +class Unpacker: """Streaming unpacker. Arguments: @@ -334,9 +315,7 @@ def __init__( if object_pairs_hook is not None and not callable(object_pairs_hook): raise TypeError("`object_pairs_hook` is not callable") if object_hook is not None and object_pairs_hook is not None: - raise TypeError( - "object_pairs_hook and object_hook are mutually " "exclusive" - ) + raise TypeError("object_pairs_hook and object_hook are mutually exclusive") if not callable(ext_hook): raise TypeError("`ext_hook` is not callable") @@ -428,20 +407,18 @@ def _read_header(self): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) + raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})") obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError( - "%s exceeds max_array_len(%s)" % (n, self._max_array_len) - ) + raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})") elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) + raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})") elif b == 0xC0: obj = None elif b == 0xC2: @@ -457,7 +434,7 @@ def _read_header(self): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError(f"{n} exceeds max_bin_len({self._max_bin_len})") obj = self._read(n) elif 0xC7 <= b <= 0xC9: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -465,7 +442,7 @@ def _read_header(self): L, n = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError(f"{L} exceeds max_ext_len({self._max_ext_len})") obj = self._read(L) elif 0xCA <= b <= 0xD3: size, fmt = _MSGPACK_HEADERS[b] @@ -478,9 +455,7 @@ def _read_header(self): elif 0xD4 <= b <= 0xD8: size, fmt, typ = _MSGPACK_HEADERS[b] if self._max_ext_len < size: - raise ValueError( - "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) - ) + raise ValueError(f"{size} exceeds max_ext_len({self._max_ext_len})") self._reserve(size + 1) n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size + 1 @@ -493,7 +468,7 @@ def _read_header(self): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) + raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})") obj = self._read(n) elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -501,16 +476,14 @@ def _read_header(self): (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: - raise ValueError( - "%s exceeds max_array_len(%s)" % (n, self._max_array_len) - ) + raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})") elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) + raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})") else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj @@ -549,17 +522,14 @@ def _unpack(self, execute=EX_CONSTRUCT): return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) - for _ in range(n) + (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) for _ in range(n) ) else: ret = {} for _ in range(n): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (str, bytes): - raise ValueError( - "%s is not allowed for map key" % str(type(key)) - ) + raise ValueError("%s is not allowed for map key" % str(type(key))) if type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) @@ -634,7 +604,7 @@ def tell(self): return self._stream_offset -class Packer(object): +class Packer: """ MessagePack Packer @@ -844,9 +814,9 @@ def _pack( continue if self._datetime and check(obj, _DateTime): - raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,)) + raise ValueError(f"Cannot serialize {obj!r} where tzinfo=None") - raise TypeError("Cannot serialize %r" % (obj,)) + raise TypeError(f"Cannot serialize {obj!r}") def pack(self, obj): try: @@ -933,7 +903,7 @@ def _pack_map_header(self, n): def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._pack_map_header(n) - for (k, v) in pairs: + for k, v in pairs: self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) diff --git a/pyproject.toml b/pyproject.toml index 195795f0..86fae1c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,8 @@ requires = [ "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ["py37"] +skip_string_normalization = true diff --git a/requirements.txt b/requirements.txt index 9f3c1a0d..88b5eb97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ -# Also declared in pyproject.toml, if updating here please also update there +# Also declared in pyproject.toml, if updating here please also update there. Cython~=0.29.30 -# dev only tools. no need to add pyproject -black==22.3.0 +# Tools required only for development. No need to add it to pyproject.toml file. +black==23.3.0 +pytest==7.3.1 +pyupgrade==3.3.2 diff --git a/setup.cfg b/setup.cfg index e3782842..d6888fcf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ project_urls = classifiers = Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 diff --git a/setup.py b/setup.py index 15ba7741..1cd1e8eb 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io import os import sys @@ -25,7 +24,7 @@ class NoCython(Exception): def cythonize(src): - sys.stderr.write("cythonize: %r\n" % (src,)) + sys.stderr.write(f"cythonize: {src!r}\n") cython_compiler.compile([src], cplus=True) @@ -36,11 +35,7 @@ def ensure_source(src): if not have_cython: raise NoCython cythonize(pyx) - elif ( - os.path.exists(pyx) - and os.stat(src).st_mtime < os.stat(pyx).st_mtime - and have_cython - ): + elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython: cythonize(pyx) return src diff --git a/test/test_buffer.py b/test/test_buffer.py index 04470584..7ee674ae 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import sys import pytest diff --git a/test/test_case.py b/test/test_case.py index 1c4e3229..c4c615e3 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import packb, unpackb def check(length, obj, use_bin_type=True): v = packb(obj, use_bin_type=use_bin_type) - assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v)) + assert len(v) == length, f"{obj!r} length should be {length!r} but get {len(v)!r}" assert unpackb(v, use_list=0, raw=not use_bin_type) == obj @@ -120,11 +119,11 @@ def test_match(): ), ({}, b"\x80"), ( - dict([(x, x) for x in range(15)]), + {x: x for x in range(15)}, b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e", ), ( - dict([(x, x) for x in range(16)]), + {x: x for x in range(16)}, b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f", ), ] diff --git a/test/test_except.py b/test/test_except.py index 745ebecb..8c0a9766 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from pytest import raises from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData diff --git a/test/test_extension.py b/test/test_extension.py index dfbe4350..9e5e6aad 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,4 +1,3 @@ -from __future__ import print_function import array import msgpack from msgpack import ExtType @@ -17,9 +16,7 @@ def p(s): assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16 # fixext 16 assert p(b"ABC") == b"\xc7\x03\x42ABC" # ext 8 assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123 # ext 16 - assert ( - p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 - ) # ext 32 + assert p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 # ext 32 def test_unpack_ext_type(): @@ -49,7 +46,7 @@ def default(obj): except AttributeError: data = obj.tostring() return ExtType(typecode, data) - raise TypeError("Unknown type object %r" % (obj,)) + raise TypeError(f"Unknown type object {obj!r}") def ext_hook(code, data): print("ext_hook called", code, data) diff --git a/test/test_format.py b/test/test_format.py index fbbc3f98..c06c87dc 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import unpackb @@ -25,9 +24,7 @@ def testFixRaw(): def testFixMap(): - check( - b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}} - ) + check(b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}) def testUnsignedInt(): diff --git a/test/test_limits.py b/test/test_limits.py index 4314c2c0..533bc112 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 -from __future__ import absolute_import, division, print_function, unicode_literals import pytest from msgpack import ( diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 63beab10..eaadef73 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import pytest from array import array diff --git a/test/test_newspec.py b/test/test_newspec.py index b7da486e..a6f4251b 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from msgpack import packb, unpackb, ExtType diff --git a/test/test_obj.py b/test/test_obj.py index 86c557cd..d3f870d9 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from pytest import raises from msgpack import packb, unpackb diff --git a/test/test_pack.py b/test/test_pack.py index 65c9cb17..2753e469 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 -from __future__ import absolute_import, division, print_function, unicode_literals from collections import OrderedDict from io import BytesIO @@ -81,9 +79,7 @@ def testPackByteArrays(): def testIgnoreUnicodeErrors(): - re = unpackb( - packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" - ) + re = unpackb(packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore") assert re == "abcdef" @@ -108,8 +104,8 @@ def testDecodeBinary(): def testPackFloat(): - assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0) - assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0) + assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(">f", 1.0) + assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(">d", 1.0) def testArraySize(sizes=[0, 5, 50, 1000]): @@ -154,7 +150,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): bio.seek(0) unpacker = Unpacker(bio, strict_map_key=False) for size in sizes: - assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) + assert unpacker.unpack() == {i: i * 2 for i in range(size)} def test_odict(): diff --git a/test/test_seq.py b/test/test_seq.py index 0d5d8065..def6630d 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io import msgpack diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index c091076b..6b138aad 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io from msgpack import Unpacker, BufferFull from msgpack import pack, packb diff --git a/test/test_stricttype.py b/test/test_stricttype.py index fe9ec6cd..9ffaff25 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from collections import namedtuple from msgpack import packb, unpackb, ExtType @@ -10,7 +8,7 @@ def test_namedtuple(): def default(o): if isinstance(o, T): return dict(o._asdict()) - raise TypeError("Unsupported type %s" % (type(o),)) + raise TypeError(f"Unsupported type {type(o)}") packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) unpacked = unpackb(packed, raw=False) @@ -23,7 +21,7 @@ def test_tuple(): def default(o): if isinstance(o, tuple): return {"__type__": "tuple", "value": list(o)} - raise TypeError("Unsupported type %s" % (type(o),)) + raise TypeError(f"Unsupported type {type(o)}") def convert(o): if o.get("__type__") == "tuple": @@ -44,9 +42,7 @@ def test_tuple_ext(): def default(o): if isinstance(o, tuple): # Convert to list and pack - payload = packb( - list(o), strict_types=True, use_bin_type=True, default=default - ) + payload = packb(list(o), strict_types=True, use_bin_type=True, default=default) return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload) raise TypeError(repr(o)) @@ -54,7 +50,7 @@ def convert(code, payload): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple return tuple(unpackb(payload, raw=False, ext_hook=convert)) - raise ValueError("Unknown Ext code {}".format(code)) + raise ValueError(f"Unknown Ext code {code}") data = packb(t, strict_types=True, use_bin_type=True, default=default) expected = unpackb(data, raw=False, ext_hook=convert) diff --git a/test/test_subtype.py b/test/test_subtype.py index d91d4553..d5a9adb3 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import packb, unpackb from collections import namedtuple diff --git a/test/test_unpack.py b/test/test_unpack.py index c7141024..bf3f960d 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -52,7 +52,7 @@ def hook(x): def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): - super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False) + super().__init__(ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: From e5249f877c18c88a0009f21097d7e48819579e60 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 21 Jul 2023 02:53:58 +0900 Subject: [PATCH 061/105] ci: add Python 3.12 and drop 3.7 --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88781025..76fcf27e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.11", "3.10", "3.9", "3.8", "3.7"] + py: ["3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -23,6 +23,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + allow-prereleases: true cache: "pip" - name: Build From 427736bbcc5553b0754616b58154ed26733103b6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 21 Jul 2023 11:11:04 +0900 Subject: [PATCH 062/105] try Cython 3.0 (#548) --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86fae1c7..dc8bbee3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=0.29.30", + "Cython~=3.0.0", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 88b5eb97..e27df0f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=0.29.30 +Cython~=3.0.0 # Tools required only for development. No need to add it to pyproject.toml file. black==23.3.0 From 7cfced51501b8e0786da5d6b499331ef2f492b29 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Aug 2023 18:09:42 +0900 Subject: [PATCH 063/105] start v1.0.6 development --- msgpack/__init__.py | 2 +- setup.cfg | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 638236fe..2540120b 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,7 +6,7 @@ version = (1, 0, 5) -__version__ = "1.0.5" +__version__ = "1.0.6dev1" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/setup.cfg b/setup.cfg index d6888fcf..c2e16721 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = msgpack -#version = attr: msgpack.__version__ -version = attr: msgpack.version +version = attr: msgpack.__version__ +#version = attr: msgpack.version license = Apache 2.0 author = Inada Naoki author_email = songofacandy@gmail.com @@ -17,11 +17,11 @@ project_urls = classifiers = Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Intended Audience :: Developers From 715126c67b2339381f5ad02f45d8fe367400c749 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Aug 2023 18:20:05 +0900 Subject: [PATCH 064/105] CI: update cibuildwheel to v2.15.0 (#551) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 6cf2fe94..c1eb2ed6 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.12.0 + uses: pypa/cibuildwheel@v2.15.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 7b75b4f36819c77a12518929fecc09d94e82f5bd Mon Sep 17 00:00:00 2001 From: TW Date: Thu, 31 Aug 2023 05:56:24 +0200 Subject: [PATCH 065/105] sphinx-related work (#554) fixes #510 --- .github/workflows/docs.yaml | 32 ++++++++++++++++++++++++++++++++ docs/_static/README.txt | 1 + docs/api.rst | 8 ++++---- docs/conf.py | 2 +- msgpack/_packer.pyx | 3 ++- msgpack/_unpacker.pyx | 6 +++--- msgpack/ext.py | 4 ++-- msgpack/fallback.py | 9 +++++---- tox.ini | 9 +++++++++ 9 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/docs.yaml create mode 100644 docs/_static/README.txt diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..a393c6b8 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,32 @@ +name: docs + +on: ["push", "pull_request"] + +jobs: + docs: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + shell: bash + run: | + pip install -r requirements.txt + make cython + pip install . + + - name: Sphinx Documentation Generator + run: | + pip install tox + tox -e sphinx diff --git a/docs/_static/README.txt b/docs/_static/README.txt new file mode 100644 index 00000000..1c70594f --- /dev/null +++ b/docs/_static/README.txt @@ -0,0 +1 @@ +Sphinx will copy the contents of docs/_static/ directory to the build location. diff --git a/docs/api.rst b/docs/api.rst index 93827e19..f5dfbbd2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,19 +5,19 @@ API reference .. autofunction:: pack -:func:`dump` is alias for :func:`pack` +``dump()`` is an alias for :func:`pack` .. autofunction:: packb -:func:`dumps` is alias for :func:`packb` +``dumps()`` is an alias for :func:`packb` .. autofunction:: unpack -:func:`load` is alias for :func:`unpack` +``load()`` is an alias for :func:`unpack` .. autofunction:: unpackb -:func:`loads` is alias for :func:`unpackb` +``loads()`` is an alias for :func:`unpackb` .. autoclass:: Packer :members: diff --git a/docs/conf.py b/docs/conf.py index 91ce77f0..1c1895c7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +#sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 074b39fc..3c398672 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -71,7 +71,8 @@ cdef class Packer(object): Packer's constructor has some keyword arguments: - :param callable default: + :param default: + When specified, it should be callable. Convert user type to builtin type that Packer supports. See also simplejson's document. diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d5dc5ea5..56126f43 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -217,7 +217,7 @@ cdef class Unpacker(object): :param file_like: File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) @@ -241,12 +241,12 @@ cdef class Unpacker(object): :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. - :param callable object_hook: + :param object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. (See also simplejson) - :param callable object_pairs_hook: + :param object_pairs_hook: When specified, it should be callable. Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) diff --git a/msgpack/ext.py b/msgpack/ext.py index 97942949..f7f2d77d 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -120,7 +120,7 @@ def from_unix(unix_sec): """Create a Timestamp from posix timestamp in seconds. :param unix_float: Posix timestamp in seconds. - :type unix_float: int or float. + :type unix_float: int or float """ seconds = int(unix_sec // 1) nanoseconds = int((unix_sec % 1) * 10**9) @@ -154,7 +154,7 @@ def to_unix_nano(self): def to_datetime(self): """Get the timestamp as a UTC datetime. - :rtype: datetime. + :rtype: `datetime.datetime` """ utc = datetime.timezone.utc return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ac1eaf4c..84b2617a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -139,7 +139,7 @@ class Unpacker: :param file_like: File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) @@ -163,12 +163,12 @@ class Unpacker: :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. - :param callable object_hook: + :param object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. (See also simplejson) - :param callable object_pairs_hook: + :param object_pairs_hook: When specified, it should be callable. Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) @@ -616,7 +616,8 @@ class Packer: Packer's constructor has some keyword arguments: - :param callable default: + :param default: + When specified, it should be callable. Convert user type to builtin type that Packer supports. See also simplejson's document. diff --git a/tox.ini b/tox.ini index 1ef2d189..369eddcf 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = {py35,py36,py37,py38}-{c,pure}, {pypy,pypy3}-pure, py34-x86, + sphinx, isolated_build = true [testenv] @@ -27,3 +28,11 @@ commands= python -c 'import sys; print(hex(sys.maxsize))' python -c 'from msgpack import _cmsgpack' py.test + + +[testenv:sphinx] +changedir = docs +deps = + sphinx +commands = + sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 423c6df265d0f964733b31a7e835fe91e4b8ea89 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 5 Sep 2023 03:51:04 +0200 Subject: [PATCH 066/105] move project metadata to pyproject.toml (#555) also: replace flake8 by ruff. --- docs/conf.py | 4 ++-- msgpack/__init__.py | 1 - msgpack/ext.py | 8 ++++---- msgpack/fallback.py | 2 +- pyproject.toml | 45 +++++++++++++++++++++++++++++++++++++++++ setup.cfg | 32 ----------------------------- setup.py | 2 -- test/test_buffer.py | 2 -- test/test_memoryview.py | 2 -- test/test_obj.py | 4 ++-- test/test_pack.py | 4 +--- test/test_seq.py | 2 +- test/test_subtype.py | 2 +- test/test_timestamp.py | 1 - 14 files changed, 57 insertions(+), 54 deletions(-) delete mode 100644 setup.cfg diff --git a/docs/conf.py b/docs/conf.py index 1c1895c7..6eb472af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,11 +11,11 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +#import os +#import sys #sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 2540120b..9a96c982 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,7 +2,6 @@ from .ext import ExtType, Timestamp import os -import sys version = (1, 0, 5) diff --git a/msgpack/ext.py b/msgpack/ext.py index f7f2d77d..02c2c430 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,6 +1,5 @@ from collections import namedtuple import datetime -import sys import struct @@ -20,8 +19,9 @@ def __new__(cls, code, data): class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. - When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python - msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`. + When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. + When using pure-Python msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and + unpack `Timestamp`. This class is immutable: Do not override seconds and nanoseconds. """ @@ -39,7 +39,7 @@ def __init__(self, seconds, nanoseconds=0): Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999. Default is 0. - Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. + Note: Negative times (before the UNIX epoch) are represented as neg. seconds + pos. ns. """ if not isinstance(seconds, int): raise TypeError("seconds must be an integer") diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 84b2617a..a174162a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -530,7 +530,7 @@ def _unpack(self, execute=EX_CONSTRUCT): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (str, bytes): raise ValueError("%s is not allowed for map key" % str(type(key))) - if type(key) is str: + if isinstance(key, str): key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: diff --git a/pyproject.toml b/pyproject.toml index dc8bbee3..f37d213c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,52 @@ requires = [ ] build-backend = "setuptools.build_meta" +[project] +name = "msgpack" +dynamic = ["version"] +license = {text="Apache 2.0"} +authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] +description = "MessagePack serializer" +readme = "README.md" +#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"] +#requires-python = ">=3.8" +classifiers = [ +# "Development Status :: 5 - Production/Stable", +# "Operating System :: OS Independent", +# "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", +] + +[project.urls] +Homepage = "https://msgpack.org/" +Documentation = "https://msgpack-python.readthedocs.io/" +Repository = "https://github.com/msgpack/msgpack-python/" +Tracker = "https://github.com/msgpack/msgpack-python/issues" +#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" + +[tool.setuptools.dynamic] +version = {attr = "msgpack.__version__"} + [tool.black] line-length = 100 target-version = ["py37"] skip_string_normalization = true + +[tool.ruff] +line-length = 100 +target-version = "py38" +ignore = [] + +[tool.ruff.per-file-ignores] +"msgpack/__init__.py" = ["F401", "F403"] +"msgpack/fallback.py" = ["E731"] +"test/test_seq.py" = ["E501"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c2e16721..00000000 --- a/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[metadata] -name = msgpack -version = attr: msgpack.__version__ -#version = attr: msgpack.version -license = Apache 2.0 -author = Inada Naoki -author_email = songofacandy@gmail.com -description = MessagePack serializer -long_description = file: README.md -long_description_content_type = text/markdown -url = https://msgpack.org/ - -project_urls = - Documentation = https://msgpack-python.readthedocs.io/ - Source = https://github.com/msgpack/msgpack-python - Tracker = https://github.com/msgpack/msgpack-python/issues - -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3.12 - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - Intended Audience :: Developers - License :: OSI Approved :: Apache Software License - -[flake8] -max_line_length = 100 - diff --git a/setup.py b/setup.py index 1cd1e8eb..7a34c8cf 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python -import io import os import sys -from glob import glob from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist diff --git a/test/test_buffer.py b/test/test_buffer.py index 7ee674ae..a3db339c 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import sys -import pytest from msgpack import packb, unpackb diff --git a/test/test_memoryview.py b/test/test_memoryview.py index eaadef73..dc319a63 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,9 +1,7 @@ #!/usr/bin/env python -import pytest from array import array from msgpack import packb, unpackb -import sys def make_array(f, data): diff --git a/test/test_obj.py b/test/test_obj.py index d3f870d9..f78bf426 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -33,7 +33,7 @@ def test_decode_pairs_hook(): prod_sum = 1 * 2 + 3 * 4 unpacked = unpackb( packed, - object_pairs_hook=lambda l: sum(k * v for k, v in l), + object_pairs_hook=lambda lst: sum(k * v for k, v in lst), use_list=1, strict_map_key=False, ) @@ -48,7 +48,7 @@ def test_only_one_obj_hook(): def test_bad_hook(): with raises(TypeError): packed = packb([3, 1 + 2j], default=lambda o: o) - unpacked = unpackb(packed, use_list=1) + unpackb(packed, use_list=1) def _arr_to_str(arr): diff --git a/test/test_pack.py b/test/test_pack.py index 2753e469..42325378 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -3,12 +3,10 @@ from collections import OrderedDict from io import BytesIO import struct -import sys import pytest -from pytest import raises, xfail -from msgpack import packb, unpackb, Unpacker, Packer, pack +from msgpack import packb, unpackb, Unpacker, Packer def check(data, use_list=False): diff --git a/test/test_seq.py b/test/test_seq.py index def6630d..16d9dde4 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): read_count = 0 for idx, o in enumerate(unpacker): - assert type(o) == bytes + assert isinstance(o, bytes) assert o == gen_binary_data(idx) read_count += 1 diff --git a/test/test_subtype.py b/test/test_subtype.py index d5a9adb3..0d1c41af 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from msgpack import packb, unpackb +from msgpack import packb from collections import namedtuple diff --git a/test/test_timestamp.py b/test/test_timestamp.py index af84a2ff..db5cc57a 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,4 @@ import pytest -import sys import datetime import msgpack from msgpack.ext import Timestamp From ef15f4a62c25114bec1db91aa4006ae2d3a9fb53 Mon Sep 17 00:00:00 2001 From: TW Date: Thu, 7 Sep 2023 14:25:07 +0200 Subject: [PATCH 067/105] add a basic .readthedocs.yaml file (#558) --- .readthedocs.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..8b5aaf16 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +# Read the Docs configuration file for Sphinx projects. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py From 41d6239c0a3bfb1daabe1a45ffdbecf4e9aa5469 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 12 Sep 2023 19:51:12 +0200 Subject: [PATCH 068/105] fix .readthedocs.yaml, fixes #559 (#560) --- .readthedocs.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b5aaf16..7447895d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,6 +7,17 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - build-essential + jobs: + pre_install: + - pip install -r requirements.txt + - make cython + +python: + install: + - method: pip + path: . sphinx: configuration: docs/conf.py From 4e10c10aaa8350f23e4b85d27ff131f7b4fd13e2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 13 Sep 2023 18:40:04 +0900 Subject: [PATCH 069/105] prepare for 1.0.6rc1 (#557) --- ChangeLog.rst | 9 +++++++++ msgpack/__init__.py | 4 ++-- pyproject.toml | 12 ++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 53547996..bf345ddb 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.6rc1 +======== + +Release Date: 2023-09-13 + +* Add Python 3.12 wheels (#517) +* Remove Python 2.7, 3.6, and 3.7 support + + 1.0.5 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 9a96c982..2e201330 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 5) -__version__ = "1.0.6dev1" +version = (1, 0, 6, "rc", 1) +__version__ = "1.0.6rc1" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/pyproject.toml b/pyproject.toml index f37d213c..a63009a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" -#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"] -#requires-python = ">=3.8" +keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] +requires-python = ">=3.8" classifiers = [ -# "Development Status :: 5 - Production/Stable", -# "Operating System :: OS Independent", -# "Programming Language :: Python", + "Development Status :: 5 - Production/Stable", + "Operating System :: OS Independent", + "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -37,7 +37,7 @@ Homepage = "https://msgpack.org/" Documentation = "https://msgpack-python.readthedocs.io/" Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" -#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} From e1d3d5d5c386b8b2fa99c812b4648f6532cab032 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 15 Sep 2023 12:02:06 +0900 Subject: [PATCH 070/105] update actions (#563) --- .github/workflows/black.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 1e28b7b5..e0917926 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -17,7 +17,7 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Black Code Formatter run: | diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a393c6b8..80bbba7d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,7 +17,7 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76fcf27e..4eb8849d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index c1eb2ed6..0412a385 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,11 +14,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 From b1b0edaeedd073f21023ec01a60bfa9da077ad2b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 21 Sep 2023 14:58:37 +0900 Subject: [PATCH 071/105] release v1.0.6 (#564) --- ChangeLog.rst | 6 +++--- msgpack/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index bf345ddb..bad51aaf 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ -1.0.6rc1 -======== +1.0.6 +===== -Release Date: 2023-09-13 +Release Date: 2023-09-21 * Add Python 3.12 wheels (#517) * Remove Python 2.7, 3.6, and 3.7 support diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 2e201330..781bcdf4 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 6, "rc", 1) -__version__ = "1.0.6rc1" +version = (1, 0, 6) +__version__ = "1.0.6" if os.environ.get("MSGPACK_PUREPYTHON"): From ecf03748c7241a0fb6bef733c7e5d2d68179b670 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 15:03:16 +0900 Subject: [PATCH 072/105] remove inline macro for msvc (#567) --- msgpack/pack.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 1e849acc..2453428c 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -26,10 +26,6 @@ extern "C" { #endif -#ifdef _MSC_VER -#define inline __inline -#endif - typedef struct msgpack_packer { char *buf; size_t length; From acd068439233b8f04543c4ee81c18c8dbb681aba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 15:25:10 +0900 Subject: [PATCH 073/105] do not fallback on build error (#568) --- setup.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 7a34c8cf..a13bd81d --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ class NoCython(Exception): def cythonize(src): + if not have_cython: + raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") cython_compiler.compile([src], cplus=True) @@ -29,31 +31,15 @@ def cythonize(src): def ensure_source(src): pyx = os.path.splitext(src)[0] + ".pyx" - if not os.path.exists(src): - if not have_cython: - raise NoCython + if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime: cythonize(pyx) - elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython: - cythonize(pyx) - return src class BuildExt(build_ext): def build_extension(self, ext): - try: - ext.sources = list(map(ensure_source, ext.sources)) - except NoCython: - print("WARNING") - print("Cython is required for building extension from checkout.") - print("Install Cython >= 0.16 or install msgpack from PyPI.") - print("Falling back to pure Python implementation.") - return - try: - return build_ext.build_extension(self, ext) - except Exception as e: - print("WARNING: Failed to compile extension modules.") - print("msgpack uses fallback pure python implementation.") - print(e) + for src in ext.sources: + ensure_source(src) + return build_ext.build_extension(self, ext) # Cython is required for sdist From 2982e9ff729eae150d67ee608fdf1d01d93d8e3f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 17:31:52 +0900 Subject: [PATCH 074/105] release v1.0.7 (#569) --- .gitignore | 1 + ChangeLog.rst | 13 +++++++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 800f1c22..8a06e267 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ MANIFEST build/* dist/* .tox +.python-version *.pyc *.pyo *.so diff --git a/ChangeLog.rst b/ChangeLog.rst index bad51aaf..ca74ebe3 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,8 +1,21 @@ +1.0.7 +===== + +Release Date: 2023-09-28 + +* Fix build error of extension module on Windows. (#567) +* ``setup.py`` doesn't skip build error of extension module. (#568) + + 1.0.6 ===== Release Date: 2023-09-21 +.. note:: + v1.0.6 Wheels for Windows don't contain extension module. + Please upgrade to v1.0.7 or newer. + * Add Python 3.12 wheels (#517) * Remove Python 2.7, 3.6, and 3.7 support diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 781bcdf4..60a088cc 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 6) -__version__ = "1.0.6" +version = (1, 0, 7) +__version__ = "1.0.7" if os.environ.get("MSGPACK_PUREPYTHON"): From c78026102c981d734c6d8fec6b2790ee31212f27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 23:34:32 +0900 Subject: [PATCH 075/105] doc: use sphinx-rtd-theme (#575) --- .github/workflows/docs.yaml | 1 - .readthedocs.yaml | 1 + docs/conf.py | 4 +--- docs/requirements.txt | 2 ++ tox.ini | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 80bbba7d..8c8298a0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -24,7 +24,6 @@ jobs: run: | pip install -r requirements.txt make cython - pip install . - name: Sphinx Documentation Generator run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7447895d..88d87182 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,6 +18,7 @@ python: install: - method: pip path: . + - requirements: docs/requirements.txt sphinx: configuration: docs/conf.py diff --git a/docs/conf.py b/docs/conf.py index 6eb472af..ab0ad3c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # msgpack documentation build configuration file, created by # sphinx-quickstart on Sun Feb 24 14:20:50 2013. # @@ -91,7 +89,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "sphinxdoc" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..8d45d0b6 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx~=7.2 +sphinx-rtd-theme~=1.3.0 diff --git a/tox.ini b/tox.ini index 369eddcf..49364be9 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,6 @@ commands= [testenv:sphinx] changedir = docs deps = - sphinx + -r docs/requirements.txt commands = sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 140864249fd0f67dffaeceeb168ffe9cdf6f1964 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 20 Dec 2023 20:46:04 +0900 Subject: [PATCH 076/105] exclude C/Cython files from wheel (#577) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a63009a7..121b1fd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,10 @@ Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +[tool.setuptools] +# Do not install C/C++/Cython source files +include-package-data = false + [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} From 039022cecb04b62a29afb8260b81f57a937aaaaa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 19:24:06 +0900 Subject: [PATCH 077/105] update Cython (#581) --- .github/workflows/wheel.yml | 10 +++++++++- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 0412a385..4a8847d0 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -3,6 +3,7 @@ on: push: branches: [main] create: + workflow_dispatch: jobs: build_wheels: @@ -23,7 +24,7 @@ jobs: platforms: arm64 - name: Set up Python 3.x - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" @@ -43,6 +44,13 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* + - name: Build pure Python wheel + env: + MSGPACK_PUREPYTHON: "1" + run: | + pip install build + python -m build -w -o wheelhouse + - name: Upload Wheels to artifact uses: actions/upload-artifact@v1 with: diff --git a/pyproject.toml b/pyproject.toml index 121b1fd9..f9af967b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=3.0.0", + "Cython~=3.0.8", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index e27df0f4..839dc5f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.0 +Cython~=3.0.8 # Tools required only for development. No need to add it to pyproject.toml file. black==23.3.0 From bf7bf88ad0b3cd7a8cd74e8251521fde743e9af9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 20:09:55 +0900 Subject: [PATCH 078/105] ci: update workflows (#582) --- .github/workflows/test.yml | 14 +++++++++++++- .github/workflows/wheel.yml | 6 +++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4eb8849d..1faeb0ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} allow-prereleases: true @@ -42,3 +42,15 @@ jobs: shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test + + - name: build packages + shell: bash + run: | + pip install build + python -m build + + - name: upload packages + uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.os }}-${{ matrix.py }} + path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 4a8847d0..e91325be 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -36,7 +36,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.15.0 + uses: pypa/cibuildwheel@v2.16.5 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -52,7 +52,7 @@ jobs: python -m build -w -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: - name: Wheels + name: wheels-${{ matrix.os }} path: wheelhouse From 9aedf8ed7f632044d42984e9710fefbd97023f71 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 20:35:28 +0900 Subject: [PATCH 079/105] Release v1.0.8 (#583) --- ChangeLog.rst | 10 ++++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index ca74ebe3..2408bc9f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.8 +===== + +Release Date: 2024-03-01 + +* Update Cython to 3.0.8. This fixes memory leak when iterating + ``Unpacker`` object on Python 3.12. +* Do not include C/Cython files in binary wheels. + + 1.0.7 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 60a088cc..919b86f1 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 7) -__version__ = "1.0.7" +version = (1, 0, 8) +__version__ = "1.0.8" if os.environ.get("MSGPACK_PUREPYTHON"): From e77672200bf89f2093b96a4abd0e4eeb253975b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?hakan=20aky=C3=BCrek?= Date: Sat, 20 Apr 2024 00:46:30 +0200 Subject: [PATCH 080/105] Avoid using floating points during timestamp-datetime conversions (#591) --- msgpack/ext.py | 6 ++++-- test/test_timestamp.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 02c2c430..3940fe0f 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -157,7 +157,9 @@ def to_datetime(self): :rtype: `datetime.datetime` """ utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( + seconds=self.seconds, microseconds=self.nanoseconds // 1000 + ) @staticmethod def from_datetime(dt): @@ -165,4 +167,4 @@ def from_datetime(dt): :rtype: Timestamp """ - return Timestamp.from_unix(dt.timestamp()) + return Timestamp(seconds=int(dt.timestamp()), nanoseconds=dt.microsecond * 1000) diff --git a/test/test_timestamp.py b/test/test_timestamp.py index db5cc57a..f9bc8353 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -86,6 +86,21 @@ def test_timestamp_datetime(): utc = datetime.timezone.utc assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) + ts = datetime.datetime(2024, 4, 16, 8, 43, 9, 420317, tzinfo=utc) + ts2 = datetime.datetime(2024, 4, 16, 8, 43, 9, 420318, tzinfo=utc) + + assert ( + Timestamp.from_datetime(ts2).nanoseconds - Timestamp.from_datetime(ts).nanoseconds == 1000 + ) + + ts3 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4256) + ts4 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4257) + assert ( + Timestamp.from_datetime(ts4).nanoseconds - Timestamp.from_datetime(ts3).nanoseconds == 1000 + ) + + assert Timestamp.from_datetime(ts).to_datetime() == ts + def test_unpack_datetime(): t = Timestamp(42, 14) From 2eca765533106cc4bbb44ecc565b1034ac50f91b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 May 2024 15:17:54 +0900 Subject: [PATCH 081/105] use ruff instead of black (#598) --- .github/workflows/docs.yaml | 17 +++++---- .github/workflows/{black.yaml => lint.yaml} | 19 +++++------ Makefile | 14 ++++++-- docs/requirements.txt | 4 +-- msgpack/fallback.py | 1 + pyproject.toml | 4 +-- requirements.txt | 7 +--- test/test_memoryview.py | 2 +- test/test_pack.py | 2 +- test/test_read_size.py | 1 + tox.ini | 38 --------------------- 11 files changed, 38 insertions(+), 71 deletions(-) rename .github/workflows/{black.yaml => lint.yaml} (56%) delete mode 100644 tox.ini diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 8c8298a0..08fc2f46 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -10,22 +10,25 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' architecture: 'x64' - - - name: Checkout - uses: actions/checkout@v4 + cache: "pip" + cache-dependency-path: | + requirements.txt + docs/requirements.txt - name: Build - shell: bash run: | pip install -r requirements.txt make cython - name: Sphinx Documentation Generator run: | - pip install tox - tox -e sphinx + pip install -r docs/requirements.txt + make docs diff --git a/.github/workflows/black.yaml b/.github/workflows/lint.yaml similarity index 56% rename from .github/workflows/black.yaml rename to .github/workflows/lint.yaml index e0917926..198cf7b5 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/lint.yaml @@ -1,25 +1,22 @@ -name: Black +name: lint on: ["push", "pull_request"] jobs: - black: + lint: # We want to run on external PRs, but not on our own internal PRs as they'll be run # by the push to the branch. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest steps: - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - architecture: 'x64' - - name: Checkout uses: actions/checkout@v4 - - name: Black Code Formatter + - name: ruff check + run: | + pipx run ruff check --diff msgpack/ test/ setup.py + + - name: ruff format run: | - pip install black==22.3.0 - black -S --diff --check msgpack/ test/ setup.py + pipx run ruff format --diff msgpack/ test/ setup.py diff --git a/Makefile b/Makefile index e4f22da4..5c1863c7 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,17 @@ PYTHON_SOURCES = msgpack test setup.py all: cython python setup.py build_ext -i -f -.PHONY: black -black: - black $(PYTHON_SOURCES) +.PHONY: format +format: + pipx run ruff format $(PYTHON_SOURCES) + +.PHONY: lint +lint: + pipx run ruff check $(PYTHON_SOURCES) + +.PHONY: doc +doc: + cd docs && sphinx-build -n -v -W --keep-going -b html -d doctrees . html .PHONY: pyupgrade pyupgrade: diff --git a/docs/requirements.txt b/docs/requirements.txt index 8d45d0b6..26002de4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx~=7.2 -sphinx-rtd-theme~=1.3.0 +sphinx~=7.3.7 +sphinx-rtd-theme~=2.0.0 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a174162a..ea4c4ced 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,4 +1,5 @@ """Fallback pure Python implementation of msgpack""" + from datetime import datetime as _DateTime import sys import struct diff --git a/pyproject.toml b/pyproject.toml index f9af967b..d99375ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,9 @@ skip_string_normalization = true [tool.ruff] line-length = 100 target-version = "py38" -ignore = [] +lint.ignore = [] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "msgpack/__init__.py" = ["F401", "F403"] "msgpack/fallback.py" = ["E731"] "test/test_seq.py" = ["E501"] diff --git a/requirements.txt b/requirements.txt index 839dc5f1..1164a941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,2 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.8 - -# Tools required only for development. No need to add it to pyproject.toml file. -black==23.3.0 -pytest==7.3.1 -pyupgrade==3.3.2 +Cython~=3.0.10 diff --git a/test/test_memoryview.py b/test/test_memoryview.py index dc319a63..eff4bca0 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -95,4 +95,4 @@ def test_multidim_memoryview(): view = memoryview(b"\00" * 6) data = view.cast(view.format, (3, 2)) packed = packb(data) - assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00' + assert packed == b"\xc4\x06\x00\x00\x00\x00\x00\x00" diff --git a/test/test_pack.py b/test/test_pack.py index 42325378..4a0ef403 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -89,7 +89,7 @@ def testStrictUnicodeUnpack(): def testIgnoreErrorsPack(): re = unpackb( - packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), + packb("abc\udc80\udcffdef", use_bin_type=True, unicode_errors="ignore"), raw=False, use_list=1, ) diff --git a/test/test_read_size.py b/test/test_read_size.py index 33a7e7dd..a7d61fd9 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,4 +1,5 @@ """Test Unpacker's read_array_header and read_map_header methods""" + from msgpack import packb, Unpacker, OutOfData UnexpectedTypeException = ValueError diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 49364be9..00000000 --- a/tox.ini +++ /dev/null @@ -1,38 +0,0 @@ -[tox] -envlist = - {py35,py36,py37,py38}-{c,pure}, - {pypy,pypy3}-pure, - py34-x86, - sphinx, -isolated_build = true - -[testenv] -deps= - pytest - -changedir=test -commands= - c,x86: python -c 'from msgpack import _cmsgpack' - c,x86: py.test - pure: py.test -setenv= - pure: MSGPACK_PUREPYTHON=x - -[testenv:py34-x86] -basepython=python3.4-x86 -deps= - pytest - -changedir=test -commands= - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - py.test - - -[testenv:sphinx] -changedir = docs -deps = - -r docs/requirements.txt -commands = - sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 0602baf3ea7fb597d1c78a90980071e03a536836 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 May 2024 18:20:09 +0900 Subject: [PATCH 082/105] update Cython and setuptools (#599) --- DEVELOP.md | 8 -------- pyproject.toml | 7 +++---- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index 9c823c34..27adf8c0 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -1,13 +1,5 @@ # Developer's note -## Wheels - -Wheels for macOS and Linux are built on Travis and AppVeyr, in -[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository. - -Wheels for Windows are built on Github Actions in this repository. - - ### Build ``` diff --git a/pyproject.toml b/pyproject.toml index d99375ac..6254f06f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,8 @@ [build-system] requires = [ - # Also declared in requirements.txt, if updating here please also update - # there - "Cython~=3.0.8", - "setuptools >= 35.0.2", + # Also declared in requirements.txt, if updating here please also update there + "Cython~=3.0.10", + "setuptools >= 69.5.1", ] build-backend = "setuptools.build_meta" From 3e9a2a7419714c294be0590aab24f2dc040581f5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:01:48 +0900 Subject: [PATCH 083/105] Stop using c++ (#600) Python 3.13a6+ & C++ & Cython cause compile error on some compilers. --- .github/workflows/test.yml | 2 +- Makefile | 2 +- msgpack/_unpacker.pyx | 2 +- msgpack/pack.h | 2 + msgpack/unpack_container_header.h | 51 +++++++++++++++++++ msgpack/unpack_template.h | 85 ++++++++----------------------- setup.py | 6 +-- 7 files changed, 81 insertions(+), 69 deletions(-) create mode 100644 msgpack/unpack_container_header.h diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1faeb0ca..530238c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} diff --git a/Makefile b/Makefile index 5c1863c7..3ce178f2 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ pyupgrade: .PHONY: cython cython: - cython --cplus msgpack/_cmsgpack.pyx + cython msgpack/_cmsgpack.pyx .PHONY: test test: cython diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 56126f43..2771e7bf 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -35,7 +35,7 @@ cdef extern from "unpack.h": PyObject* timestamp_t PyObject *giga; PyObject *utc; - char *unicode_errors + const char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len Py_ssize_t max_array_len diff --git a/msgpack/pack.h b/msgpack/pack.h index 2453428c..901fad7b 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -24,6 +24,8 @@ #ifdef __cplusplus extern "C" { +#else +#define bool char #endif typedef struct msgpack_packer { diff --git a/msgpack/unpack_container_header.h b/msgpack/unpack_container_header.h new file mode 100644 index 00000000..c14a3c2b --- /dev/null +++ b/msgpack/unpack_container_header.h @@ -0,0 +1,51 @@ +static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +{ + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; + +#define inc_offset(inc) \ + if (len - *off < inc) \ + return 0; \ + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; +#ifdef USE_CASE_RANGE + case fixed_offset + 0x0 ... fixed_offset + 0xf: +#else + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: +#endif + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); + return 1; +} + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8b9fcc19..cce29e7a 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -75,8 +75,7 @@ static inline void unpack_clear(unpack_context *ctx) Py_CLEAR(ctx->stack[0].obj); } -template -static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); @@ -386,6 +385,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize #undef construct_cb } +#undef NEXT_CS #undef SWITCH_RANGE_BEGIN #undef SWITCH_RANGE #undef SWITCH_RANGE_DEFAULT @@ -397,68 +397,27 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize #undef again_fixed_trail_if_zero #undef start_container -template -static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) -{ - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; - -#define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; - - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; -#ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: -#else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: -#endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); - return 1; +static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(1, ctx, data, len, off); +} +static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(0, ctx, data, len, off); } -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END - -static const execute_fn unpack_construct = &unpack_execute; -static const execute_fn unpack_skip = &unpack_execute; -static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>; -static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; - -#undef NEXT_CS +#define unpack_container_header read_array_header +#define fixed_offset 0x90 +#define var_offset 0xdc +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset + +#define unpack_container_header read_map_header +#define fixed_offset 0x80 +#define var_offset 0xde +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset /* vim: set ts=4 sw=4 sts=4 expandtab */ diff --git a/setup.py b/setup.py index a13bd81d..dc14a26e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def cythonize(src): if not have_cython: raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src], cplus=True) + cython_compiler.compile([src]) def ensure_source(src): @@ -51,17 +51,17 @@ def __init__(self, *args, **kwargs): libraries = [] macros = [] +ext_modules = [] if sys.platform == "win32": libraries.append("ws2_32") macros = [("__LITTLE_ENDIAN__", "1")] -ext_modules = [] if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", - sources=["msgpack/_cmsgpack.cpp"], + sources=["msgpack/_cmsgpack.c"], libraries=libraries, include_dirs=["."], define_macros=macros, From b389ccf2f72355e23836ced193b555401508ef81 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:10:37 +0900 Subject: [PATCH 084/105] update README (#561) --- README.md | 116 +++++++++++++++++++++++------------------------------- 1 file changed, 50 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 61f99e1f..d3247bdf 100644 --- a/README.md +++ b/README.md @@ -10,53 +10,6 @@ It lets you exchange data among multiple languages like JSON. But it's faster and smaller. This package provides CPython bindings for reading and writing MessagePack data. - -## Very important notes for existing users - -### PyPI package name - -Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. - -When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before -`pip install -U msgpack`. - - -### Compatibility with the old format - -You can use `use_bin_type=False` option to pack `bytes` -object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. - -You can unpack old msgpack format using `raw=True` option. -It unpacks str (raw) type in msgpack into Python bytes. - -See note below for detail. - - -### Major breaking changes in msgpack 1.0 - -* Python 2 - - * The extension module does not support Python 2 anymore. - The pure Python implementation (`msgpack.fallback`) is used for Python 2. - -* Packer - - * `use_bin_type=True` by default. bytes are encoded in bin type in msgpack. - **If you are still using Python 2, you must use unicode for all string types.** - You can use `use_bin_type=False` to encode into old msgpack format. - * `encoding` option is removed. UTF-8 is used always. - -* Unpacker - - * `raw=False` by default. It assumes str types are valid UTF-8 string - and decode them to Python str (unicode) object. - * `encoding` option is removed. You can use `raw=True` to support old format. - * Default value of `max_buffer_size` is changed from 0 to 100 MiB. - * Default value of `strict_map_key` is changed to True to avoid hashdos. - You need to pass `strict_map_key=False` if you have data which contain map keys - which type is not bytes or str. - - ## Install ``` @@ -65,12 +18,9 @@ $ pip install msgpack ### Pure Python implementation -The extension module in msgpack (`msgpack._cmsgpack`) does not support -Python 2 and PyPy. - -But msgpack provides a pure Python implementation (`msgpack.fallback`) -for PyPy and Python 2. +The extension module in msgpack (`msgpack._cmsgpack`) does not support PyPy. +But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy. ### Windows @@ -82,10 +32,6 @@ Without extension, using pure Python implementation on CPython runs slowly. ## How to use -NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users -using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them. - - ### One-shot pack & unpack Use `packb` for packing and `unpackb` for unpacking. @@ -97,16 +43,16 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with ```pycon >>> import msgpack ->>> msgpack.packb([1, 2, 3], use_bin_type=True) +>>> msgpack.packb([1, 2, 3]) '\x93\x01\x02\x03' ->>> msgpack.unpackb(_, raw=False) +>>> msgpack.unpackb(_) [1, 2, 3] ``` `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: ```pycon ->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) +>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) (1, 2, 3) ``` @@ -127,11 +73,11 @@ from io import BytesIO buf = BytesIO() for i in range(100): - buf.write(msgpack.packb(i, use_bin_type=True)) + buf.write(msgpack.packb(i)) buf.seek(0) -unpacker = msgpack.Unpacker(buf, raw=False) +unpacker = msgpack.Unpacker(buf) for unpacked in unpacker: print(unpacked) ``` @@ -162,8 +108,8 @@ def encode_datetime(obj): return obj -packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) -this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) +packed_dict = msgpack.packb(useful_dict, default=encode_datetime) +this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) ``` `Unpacker`'s `object_hook` callback receives a dict; the @@ -191,8 +137,8 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... return ExtType(code, data) ... >>> data = array.array('d', [1.2, 3.4]) ->>> packed = msgpack.packb(data, default=default, use_bin_type=True) ->>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) +>>> packed = msgpack.packb(data, default=default) +>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook) >>> data == unpacked True ``` @@ -210,7 +156,7 @@ in a map, can be unpacked or skipped individually. ## Notes -### string and binary type +### string and binary type in old msgpack spec Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -263,3 +209,41 @@ You can use `gc.disable()` when unpacking large message. List is the default sequence type of Python. But tuple is lighter than list. You can use `use_list=False` while unpacking when performance is important. + + +## Major breaking changes in the history + +### msgpack 0.5 + +Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. + +When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before +`pip install -U msgpack`. + + +### msgpack 1.0 + +* Python 2 support + + * The extension module does not support Python 2 anymore. + The pure Python implementation (`msgpack.fallback`) is used for Python 2. + + * msgpack 1.0.6 drops official support of Python 2.7, as pip and + GitHub Action (setup-python) no longer support Python 2.7. + +* Packer + + * Packer uses `use_bin_type=True` by default. + Bytes are encoded in bin type in msgpack. + * The `encoding` option is removed. UTF-8 is used always. + +* Unpacker + + * Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string + and decode them to Python str (unicode) object. + * `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str). + * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack. + You need to pass `max_buffer_size=0` if you have large but safe data. + * Default value of `strict_map_key` is changed to True to avoid hashdos. + You need to pass `strict_map_key=False` if you have data which contain map keys + which type is not bytes or str. From 526ec9c923c4867c96537c1f09783fe59432f737 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:49:22 +0900 Subject: [PATCH 085/105] update cibuildwheel to 2.17 (#601) --- .github/workflows/docs.yaml | 1 - .github/workflows/wheel.yml | 15 ++++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 08fc2f46..b696b926 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,7 +17,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.x' - architecture: 'x64' cache: "pip" cache-dependency-path: | requirements.txt diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index e91325be..d57e0586 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,29 +14,25 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 with: - platforms: arm64 + platforms: all - - name: Set up Python 3.x - uses: actions/setup-python@v5 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" - - - name: Prepare + - name: Cythonize shell: bash run: | pip install -r requirements.txt make cython - name: Build - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -45,6 +41,7 @@ jobs: CIBW_SKIP: pp* - name: Build pure Python wheel + if: runner.os == 'Linux' env: MSGPACK_PUREPYTHON: "1" run: | From 52f8bc2e557ca6684f2f73d129da8317a88bc431 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 5 May 2024 23:14:27 +0900 Subject: [PATCH 086/105] implement buffer protocol (#602) Fix #479 --- msgpack/_packer.pyx | 10 +++++++--- msgpack/buff_converter.h | 8 -------- 2 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 msgpack/buff_converter.h diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 3c398672..c2018806 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -44,8 +44,6 @@ cdef extern from "pack.h": int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) -cdef extern from "buff_converter.h": - object buff_to_buff(char *, Py_ssize_t) cdef int DEFAULT_RECURSE_LIMIT=511 cdef long long ITEM_LIMIT = (2**32)-1 @@ -371,4 +369,10 @@ cdef class Packer(object): def getbuffer(self): """Return view of internal buffer.""" - return buff_to_buff(self.pk.buf, self.pk.length) + return memoryview(self) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) + + def __releasebuffer__(self, Py_buffer *buffer): + pass diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h deleted file mode 100644 index 86b4196d..00000000 --- a/msgpack/buff_converter.h +++ /dev/null @@ -1,8 +0,0 @@ -#include "Python.h" - -/* cython does not support this preprocessor check => write it in raw C */ -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - return PyMemoryView_FromMemory(buff, size, PyBUF_READ); -} From a97b31437d3301fa4c29b7813dabee0e690756e8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:13:59 +0900 Subject: [PATCH 087/105] Remove unused code (#603) --- msgpack/_packer.pyx | 7 -- msgpack/pack.h | 3 +- msgpack/pack_template.h | 238 ++-------------------------------------- 3 files changed, 8 insertions(+), 240 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c2018806..c7eeda11 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -16,8 +16,6 @@ from .ext import ExtType, Timestamp cdef extern from "Python.h": int PyMemoryView_Check(object obj) - char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL - cdef extern from "pack.h": struct msgpack_packer: @@ -26,11 +24,9 @@ cdef extern from "pack.h": size_t buf_size bint use_bin_type - int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long(msgpack_packer* pk, long d) int msgpack_pack_long_long(msgpack_packer* pk, long long d) int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) int msgpack_pack_float(msgpack_packer* pk, float d) @@ -184,9 +180,6 @@ cdef class Packer(object): continue else: raise OverflowError("Integer value out of range") - elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - longval = o - ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): if self.use_float: fval = o diff --git a/msgpack/pack.h b/msgpack/pack.h index 901fad7b..688eab86 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -21,11 +21,10 @@ #include "sysdep.h" #include #include +#include #ifdef __cplusplus extern "C" { -#else -#define bool char #endif typedef struct msgpack_packer { diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 7d479b6d..b8959f02 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -37,18 +37,6 @@ * Integer */ -#define msgpack_pack_real_uint8(x, d) \ -do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ -} while(0) - #define msgpack_pack_real_uint16(x, d) \ do { \ if(d < (1<<7)) { \ @@ -123,18 +111,6 @@ do { \ } \ } while(0) -#define msgpack_pack_real_int8(x, d) \ -do { \ - if(d < -(1<<5)) { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } \ -} while(0) - #define msgpack_pack_real_int16(x, d) \ do { \ if(d < -(1<<5)) { \ @@ -264,49 +240,6 @@ do { \ } while(0) -static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d) -{ - msgpack_pack_real_uint8(x, d); -} - -static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d) -{ - msgpack_pack_real_uint16(x, d); -} - -static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d) -{ - msgpack_pack_real_uint32(x, d); -} - -static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d) -{ - msgpack_pack_real_uint64(x, d); -} - -static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d) -{ - msgpack_pack_real_int8(x, d); -} - -static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d) -{ - msgpack_pack_real_int16(x, d); -} - -static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d) -{ - msgpack_pack_real_int32(x, d); -} - -static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) -{ - msgpack_pack_real_int64(x, d); -} - - -//#ifdef msgpack_pack_inline_func_cint - static inline int msgpack_pack_short(msgpack_packer* x, short d) { #if defined(SIZEOF_SHORT) @@ -372,192 +305,37 @@ if(sizeof(int) == 2) { static inline int msgpack_pack_long(msgpack_packer* x, long d) { #if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 +#if SIZEOF_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif #elif defined(LONG_MAX) -#if LONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LONG_MAX == 0x7fffffffL +#if LONG_MAX == 0x7fffffffL msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif #else -if(sizeof(long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} + if (sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); + } else { + msgpack_pack_real_int64(x, d); + } #endif } static inline int msgpack_pack_long_long(msgpack_packer* x, long long d) { -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#elif defined(LLONG_MAX) -#if LLONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LLONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(long long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long long) == 4) { - msgpack_pack_real_int32(x, d); -} else { msgpack_pack_real_int64(x, d); } -#endif -} - -static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d) -{ -#if defined(SIZEOF_SHORT) -#if SIZEOF_SHORT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(USHRT_MAX) -#if USHRT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif USHRT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned short) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned short) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d) -{ -#if defined(SIZEOF_INT) -#if SIZEOF_INT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(UINT_MAX) -#if UINT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif UINT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned int) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d) -{ -#if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULONG_MAX) -#if ULONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d) { -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULLONG_MAX) -#if ULLONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULLONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { msgpack_pack_real_uint64(x, d); } -#endif -} - -//#undef msgpack_pack_inline_func_cint -//#endif - /* @@ -810,11 +588,9 @@ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uin #undef TAKE8_32 #undef TAKE8_64 -#undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 #undef msgpack_pack_real_uint32 #undef msgpack_pack_real_uint64 -#undef msgpack_pack_real_int8 #undef msgpack_pack_real_int16 #undef msgpack_pack_real_int32 #undef msgpack_pack_real_int64 From bf2413f915474841aafee2c98321dc465e5f0a3e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:30:07 +0900 Subject: [PATCH 088/105] ignore msgpack/*.c --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a06e267..341be631 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist/* *.so *~ msgpack/__version__.py +msgpack/*.c msgpack/*.cpp *.egg-info /venv From 72e65feb0e02449fa191346f26b54d3842ab7e69 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:49:12 +0900 Subject: [PATCH 089/105] packer: add `buf_size` option (#604) And change the default buffer size to 256KiB. Signed-off-by: Rodrigo Tobar Co-authored-by: Rodrigo Tobar --- msgpack/_packer.pyx | 13 +++++++++---- msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 27 +++++---------------------- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c7eeda11..99557d37 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -53,7 +53,7 @@ cdef inline int PyBytesLike_CheckExact(object o): return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) -cdef class Packer(object): +cdef class Packer: """ MessagePack Packer @@ -97,6 +97,11 @@ cdef class Packer(object): :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. + + :param int buf_size: + The size of the internal buffer. (default: 256*1024) + Useful if serialisation size can be correctly estimated, + avoid unnecessary reallocations. """ cdef msgpack_packer pk cdef object _default @@ -107,8 +112,7 @@ cdef class Packer(object): cdef bint autoreset cdef bint datetime - def __cinit__(self): - cdef int buf_size = 1024*1024 + def __cinit__(self, buf_size=256*1024, **_kwargs): self.pk.buf = PyMem_Malloc(buf_size) if self.pk.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") @@ -117,7 +121,8 @@ cdef class Packer(object): def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, - bint strict_types=False, bint datetime=False, unicode_errors=None): + bint strict_types=False, bint datetime=False, unicode_errors=None, + buf_size=256*1024): self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2771e7bf..34ff3304 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -210,7 +210,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise ValueError("Unpack failed: error = %d" % (ret,)) -cdef class Unpacker(object): +cdef class Unpacker: """Streaming unpacker. Arguments: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ea4c4ced..cbf0d30e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -232,6 +232,7 @@ class Unpacker: def __init__( self, file_like=None, + *, read_size=0, use_list=True, raw=False, @@ -650,32 +651,13 @@ class Packer: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. - Example of streaming deserialize from file-like object:: - - unpacker = Unpacker(file_like) - for o in unpacker: - process(o) - - Example of streaming deserialize from socket:: - - unpacker = Unpacker() - while True: - buf = sock.recv(1024**2) - if not buf: - break - unpacker.feed(buf) - for o in unpacker: - process(o) - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``OutOfData`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. + :param int buf_size: + Internal buffer size. This option is used only for C implementation. """ def __init__( self, + *, default=None, use_single_float=False, autoreset=True, @@ -683,6 +665,7 @@ def __init__( strict_types=False, datetime=False, unicode_errors=None, + buf_size=None, ): self._strict_types = strict_types self._use_float = use_single_float From 3da5818a3a96a4bd39f25a483ee3c22ecb626e9d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 02:12:46 +0900 Subject: [PATCH 090/105] update readme (#605) --- README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d3247bdf..61a03c60 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,7 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with [1, 2, 3] ``` -`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: - -```pycon ->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) -(1, 2, 3) -``` - -You should always specify the `use_list` keyword argument for backward compatibility. -See performance issues relating to `use_list option`_ below. - -Read the docstring for other options. +Read the docstring for options. ### Streaming unpacking @@ -116,6 +106,9 @@ this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) `object_pairs_hook` callback may instead be used to receive a list of key-value pairs. +NOTE: msgpack can encode datetime with tzinfo into standard ext type for now. +See `datetime` option in `Packer` docstring. + ### Extended types From e1068087e0bb3aebd77c4f2407bf9c429f631787 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 02:13:12 +0900 Subject: [PATCH 091/105] cython: better exception handling (#606) - use `except -1` instead of manual error handling - use `PyUnicode_AsUTF8AndSize()` - use `_pack()` and `_pack_inner()` instead of `while True:` --- msgpack/_packer.pyx | 302 +++++++++++++++++++------------------------- msgpack/pack.h | 21 --- 2 files changed, 131 insertions(+), 192 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 99557d37..ad532215 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -24,21 +24,20 @@ cdef extern from "pack.h": size_t buf_size bint use_bin_type - int msgpack_pack_nil(msgpack_packer* pk) - int msgpack_pack_true(msgpack_packer* pk) - int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long_long(msgpack_packer* pk, long long d) - int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) - int msgpack_pack_float(msgpack_packer* pk, float d) - int msgpack_pack_double(msgpack_packer* pk, double d) - int msgpack_pack_array(msgpack_packer* pk, size_t l) - int msgpack_pack_map(msgpack_packer* pk, size_t l) - int msgpack_pack_raw(msgpack_packer* pk, size_t l) - int msgpack_pack_bin(msgpack_packer* pk, size_t l) - int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) - int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) - int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); - int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) + int msgpack_pack_nil(msgpack_packer* pk) except -1 + int msgpack_pack_true(msgpack_packer* pk) except -1 + int msgpack_pack_false(msgpack_packer* pk) except -1 + int msgpack_pack_long_long(msgpack_packer* pk, long long d) except -1 + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) except -1 + int msgpack_pack_float(msgpack_packer* pk, float d) except -1 + int msgpack_pack_double(msgpack_packer* pk, double d) except -1 + int msgpack_pack_array(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_map(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_raw(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_bin(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) except -1 + int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) except -1 + int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds) except -1 cdef int DEFAULT_RECURSE_LIMIT=511 @@ -119,6 +118,10 @@ cdef class Packer: self.pk.buf_size = buf_size self.pk.length = 0 + def __dealloc__(self): + PyMem_Free(self.pk.buf) + self.pk.buf = NULL + def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False, bint datetime=False, unicode_errors=None, @@ -139,155 +142,127 @@ cdef class Packer: else: self.unicode_errors = self._berrors - def __dealloc__(self): - PyMem_Free(self.pk.buf) - self.pk.buf = NULL - - cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: + # returns -2 when default should(o) be called + cdef int _pack_inner(self, object o, bint will_default, int nest_limit) except -1: cdef long long llval cdef unsigned long long ullval cdef unsigned long ulval - cdef long longval - cdef float fval - cdef double dval - cdef char* rawval - cdef int ret - cdef dict d + cdef const char* rawval cdef Py_ssize_t L - cdef int default_used = 0 - cdef bint strict_types = self.strict_types + cdef bool strict_types = self.strict_types cdef Py_buffer view - if nest_limit < 0: - raise ValueError("recursion limit exceeded.") - - while True: - if o is None: - ret = msgpack_pack_nil(&self.pk) - elif o is True: - ret = msgpack_pack_true(&self.pk) - elif o is False: - ret = msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): - # PyInt_Check(long) is True for Python 3. - # So we should test long before int. - try: - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) - except OverflowError as oe: - if not default_used and self._default is not None: - o = self._default(o) - default_used = True - continue - else: - raise OverflowError("Integer value out of range") - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - if self.use_float: - fval = o - ret = msgpack_pack_float(&self.pk, fval) + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): + try: + if o > 0: + ullval = o + msgpack_pack_unsigned_long_long(&self.pk, ullval) else: - dval = o - ret = msgpack_pack_double(&self.pk, dval) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - L = Py_SIZE(o) - if L > ITEM_LIMIT: - PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - rawval = o - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if self.unicode_errors == NULL: - ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - if ret == -2: - raise ValueError("unicode string is too large") + llval = o + msgpack_pack_long_long(&self.pk, llval) + except OverflowError as oe: + if will_default: + return -2 else: - o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) - L = Py_SIZE(o) - if L > ITEM_LIMIT: - raise ValueError("unicode string is too large") - ret = msgpack_pack_raw(&self.pk, L) - if ret == 0: - rawval = o - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o): - d = o - L = len(d) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in d.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif not strict_types and PyDict_Check(o): - L = len(o) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in o.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - # This should be before Tuple because ExtType is namedtuple. - longval = o.code - rawval = o.data - L = len(o.data) - if L > ITEM_LIMIT: - raise ValueError("EXT data is too large") - ret = msgpack_pack_ext(&self.pk, longval, L) - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif type(o) is Timestamp: - llval = o.seconds - ulval = o.nanoseconds - ret = msgpack_pack_timestamp(&self.pk, llval, ulval) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + raise OverflowError("Integer value out of range") + elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): + if self.use_float: + msgpack_pack_float(&self.pk, o) + else: + msgpack_pack_double(&self.pk, o) + elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): + L = Py_SIZE(o) + if L > ITEM_LIMIT: + PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) + rawval = o + msgpack_pack_bin(&self.pk, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): + if self.unicode_errors == NULL: + rawval = PyUnicode_AsUTF8AndSize(o, &L) + if L >ITEM_LIMIT: + raise ValueError("unicode string is too large") + else: + o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) L = Py_SIZE(o) if L > ITEM_LIMIT: - raise ValueError("list is too large") - ret = msgpack_pack_array(&self.pk, L) - if ret == 0: - for v in o: - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif PyMemoryView_Check(o): - if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - raise ValueError("could not get buffer for memoryview") - L = view.len - if L > ITEM_LIMIT: - PyBuffer_Release(&view); - raise ValueError("memoryview is too large") - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, view.buf, L) + raise ValueError("unicode string is too large") + rawval = o + msgpack_pack_raw(&self.pk, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o): + L = len(o) + if L > ITEM_LIMIT: + raise ValueError("dict is too large") + msgpack_pack_map(&self.pk, L) + for k, v in o.items(): + self._pack(k, nest_limit) + self._pack(v, nest_limit) + elif type(o) is ExtType if strict_types else isinstance(o, ExtType): + # This should be before Tuple because ExtType is namedtuple. + rawval = o.data + L = len(o.data) + if L > ITEM_LIMIT: + raise ValueError("EXT data is too large") + msgpack_pack_ext(&self.pk, o.code, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif type(o) is Timestamp: + llval = o.seconds + ulval = o.nanoseconds + msgpack_pack_timestamp(&self.pk, llval, ulval) + elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + L = Py_SIZE(o) + if L > ITEM_LIMIT: + raise ValueError("list is too large") + msgpack_pack_array(&self.pk, L) + for v in o: + self._pack(v, nest_limit) + elif PyMemoryView_Check(o): + PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) + L = view.len + if L > ITEM_LIMIT: PyBuffer_Release(&view); - elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: - delta = o - epoch - if not PyDelta_CheckExact(delta): - raise ValueError("failed to calculate delta") - llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) - ulval = timedelta_microseconds(delta) * 1000 - ret = msgpack_pack_timestamp(&self.pk, llval, ulval) - elif not default_used and self._default: + raise ValueError("memoryview is too large") + try: + msgpack_pack_bin(&self.pk, L) + msgpack_pack_raw_body(&self.pk, view.buf, L) + finally: + PyBuffer_Release(&view); + elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: + delta = o - epoch + if not PyDelta_CheckExact(delta): + raise ValueError("failed to calculate delta") + llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) + ulval = timedelta_microseconds(delta) * 1000 + msgpack_pack_timestamp(&self.pk, llval, ulval) + elif will_default: + return -2 + elif self.datetime and PyDateTime_CheckExact(o): + # this should be later than will_default + PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) + else: + PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) + + cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: + cdef int ret + if nest_limit < 0: + raise ValueError("recursion limit exceeded.") + nest_limit -= 1 + if self._default is not None: + ret = self._pack_inner(o, 1, nest_limit) + if ret == -2: o = self._default(o) - default_used = 1 - continue - elif self.datetime and PyDateTime_CheckExact(o): - PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) else: - PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) - return ret + return ret + return self._pack_inner(o, 0, nest_limit) - cpdef pack(self, object obj): + def pack(self, object obj): cdef int ret try: ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) @@ -308,11 +283,7 @@ cdef class Packer: def pack_array_header(self, long long size): if size > ITEM_LIMIT: raise ValueError - cdef int ret = msgpack_pack_array(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_array(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 @@ -321,11 +292,7 @@ cdef class Packer: def pack_map_header(self, long long size): if size > ITEM_LIMIT: raise ValueError - cdef int ret = msgpack_pack_map(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_map(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 @@ -338,17 +305,10 @@ cdef class Packer: *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ - cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - if ret == 0: - for k, v in pairs: - ret = self._pack(k) - if ret != 0: break - ret = self._pack(v) - if ret != 0: break - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_map(&self.pk, len(pairs)) + for k, v in pairs: + self._pack(k) + self._pack(v) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 diff --git a/msgpack/pack.h b/msgpack/pack.h index 688eab86..edf3a3fe 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -64,27 +64,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ #include "pack_template.h" -// return -2 when o is too long -static inline int -msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) -{ - assert(PyUnicode_Check(o)); - - Py_ssize_t len; - const char* buf = PyUnicode_AsUTF8AndSize(o, &len); - if (buf == NULL) - return -1; - - if (len > limit) { - return -2; - } - - int ret = msgpack_pack_raw(pk, len); - if (ret) return ret; - - return msgpack_pack_raw_body(pk, buf, len); -} - #ifdef __cplusplus } #endif From e0f0e145f15364819bac80bd0808834a9df0065e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 03:33:48 +0900 Subject: [PATCH 092/105] better error checks (#607) * check buffer exports * add error messages --- msgpack/_packer.pyx | 48 +++++++++++++++++++++++++++++++++------------ test/test_buffer.py | 24 +++++++++++++++++++++-- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index ad532215..402b6946 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -106,6 +106,7 @@ cdef class Packer: cdef object _default cdef object _berrors cdef const char *unicode_errors + cdef size_t exports # number of exported buffers cdef bint strict_types cdef bint use_float cdef bint autoreset @@ -117,10 +118,16 @@ cdef class Packer: raise MemoryError("Unable to allocate internal buffer.") self.pk.buf_size = buf_size self.pk.length = 0 + self.exports = 0 def __dealloc__(self): PyMem_Free(self.pk.buf) self.pk.buf = NULL + assert self.exports == 0 + + cdef _check_exports(self): + if self.exports > 0: + raise BufferError("Existing exports of data: Packer cannot be changed") def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, @@ -149,8 +156,8 @@ cdef class Packer: cdef unsigned long ulval cdef const char* rawval cdef Py_ssize_t L - cdef bool strict_types = self.strict_types cdef Py_buffer view + cdef bint strict = self.strict_types if o is None: msgpack_pack_nil(&self.pk) @@ -158,7 +165,7 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): + elif PyLong_CheckExact(o) if strict else PyLong_Check(o): try: if o > 0: ullval = o @@ -171,19 +178,19 @@ cdef class Packer: return -2 else: raise OverflowError("Integer value out of range") - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): + elif PyFloat_CheckExact(o) if strict else PyFloat_Check(o): if self.use_float: msgpack_pack_float(&self.pk, o) else: msgpack_pack_double(&self.pk, o) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): + elif PyBytesLike_CheckExact(o) if strict else PyBytesLike_Check(o): L = Py_SIZE(o) if L > ITEM_LIMIT: PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) rawval = o msgpack_pack_bin(&self.pk, L) msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): + elif PyUnicode_CheckExact(o) if strict else PyUnicode_Check(o): if self.unicode_errors == NULL: rawval = PyUnicode_AsUTF8AndSize(o, &L) if L >ITEM_LIMIT: @@ -196,7 +203,7 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, L) msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o): + elif PyDict_CheckExact(o) if strict else PyDict_Check(o): L = len(o) if L > ITEM_LIMIT: raise ValueError("dict is too large") @@ -204,7 +211,7 @@ cdef class Packer: for k, v in o.items(): self._pack(k, nest_limit) self._pack(v, nest_limit) - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): + elif type(o) is ExtType if strict else isinstance(o, ExtType): # This should be before Tuple because ExtType is namedtuple. rawval = o.data L = len(o.data) @@ -216,7 +223,7 @@ cdef class Packer: llval = o.seconds ulval = o.nanoseconds msgpack_pack_timestamp(&self.pk, llval, ulval) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + elif PyList_CheckExact(o) if strict else (PyTuple_Check(o) or PyList_Check(o)): L = Py_SIZE(o) if L > ITEM_LIMIT: raise ValueError("list is too large") @@ -264,6 +271,7 @@ cdef class Packer: def pack(self, object obj): cdef int ret + self._check_exports() try: ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) except: @@ -277,12 +285,16 @@ cdef class Packer: return buf def pack_ext_type(self, typecode, data): + self._check_exports() + if len(data) > ITEM_LIMIT: + raise ValueError("ext data too large") msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, long long size): + self._check_exports() if size > ITEM_LIMIT: - raise ValueError + raise ValueError("array too large") msgpack_pack_array(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -290,8 +302,9 @@ cdef class Packer: return buf def pack_map_header(self, long long size): + self._check_exports() if size > ITEM_LIMIT: - raise ValueError + raise ValueError("map too learge") msgpack_pack_map(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -305,7 +318,11 @@ cdef class Packer: *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ - msgpack_pack_map(&self.pk, len(pairs)) + self._check_exports() + size = len(pairs) + if size > ITEM_LIMIT: + raise ValueError("map too large") + msgpack_pack_map(&self.pk, size) for k, v in pairs: self._pack(k) self._pack(v) @@ -319,6 +336,7 @@ cdef class Packer: This method is useful only when autoreset=False. """ + self._check_exports() self.pk.length = 0 def bytes(self): @@ -326,11 +344,15 @@ cdef class Packer: return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) def getbuffer(self): - """Return view of internal buffer.""" + """Return memoryview of internal buffer. + + Note: Packer now supports buffer protocol. You can use memoryview(packer). + """ return memoryview(self) def __getbuffer__(self, Py_buffer *buffer, int flags): PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) + self.exports += 1 def __releasebuffer__(self, Py_buffer *buffer): - pass + self.exports -= 1 diff --git a/test/test_buffer.py b/test/test_buffer.py index a3db339c..2165eb50 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +from pytest import raises -from msgpack import packb, unpackb +from msgpack import packb, unpackb, Packer def test_unpack_buffer(): @@ -27,3 +27,23 @@ def test_unpack_memoryview(): assert [b"foo", b"bar"] == obj expected_type = bytes assert all(type(s) == expected_type for s in obj) + + +def test_packer_getbuffer(): + packer = Packer(autoreset=False) + packer.pack_array_header(2) + packer.pack(42) + packer.pack("hello") + buffer = packer.getbuffer() + assert isinstance(buffer, memoryview) + assert bytes(buffer) == b"\x92*\xa5hello" + + if Packer.__module__ == "msgpack._cmsgpack": # only for Cython + # cython Packer supports buffer protocol directly + assert bytes(packer) == b"\x92*\xa5hello" + + with raises(BufferError): + packer.pack(42) + buffer.release() + packer.pack(42) + assert bytes(packer) == b"\x92*\xa5hello*" From 33e0e86f4e66e6bb029f118e8f5e166b55e828f7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 11:46:31 +0900 Subject: [PATCH 093/105] Cleanup code and pyproject (#608) * use isort * fallback: use BytesIO instead of StringIO. We had dropped Python 2 already. --- Makefile | 4 ++-- msgpack/__init__.py | 12 +++++----- msgpack/ext.py | 2 +- msgpack/fallback.py | 52 ++++++++++++++++++----------------------- pyproject.toml | 17 +++++--------- setup.py | 4 ++-- test/test_buffer.py | 2 +- test/test_except.py | 5 ++-- test/test_extension.py | 1 + test/test_limits.py | 8 +++---- test/test_memoryview.py | 1 + test/test_newspec.py | 2 +- test/test_obj.py | 1 + test/test_pack.py | 4 ++-- test/test_read_size.py | 2 +- test/test_seq.py | 6 ++--- test/test_sequnpack.py | 7 +++--- test/test_stricttype.py | 3 ++- test/test_subtype.py | 3 ++- test/test_timestamp.py | 4 +++- test/test_unpack.py | 11 ++++----- 21 files changed, 73 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index 3ce178f2..51f3e0ef 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,11 @@ all: cython .PHONY: format format: - pipx run ruff format $(PYTHON_SOURCES) + ruff format $(PYTHON_SOURCES) .PHONY: lint lint: - pipx run ruff check $(PYTHON_SOURCES) + ruff check $(PYTHON_SOURCES) .PHONY: doc doc: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 919b86f1..e796efb0 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,20 +1,20 @@ -from .exceptions import * -from .ext import ExtType, Timestamp - +# ruff: noqa: F401 import os +from .exceptions import * # noqa: F403 +from .ext import ExtType, Timestamp version = (1, 0, 8) __version__ = "1.0.8" if os.environ.get("MSGPACK_PUREPYTHON"): - from .fallback import Packer, unpackb, Unpacker + from .fallback import Packer, Unpacker, unpackb else: try: - from ._cmsgpack import Packer, unpackb, Unpacker + from ._cmsgpack import Packer, Unpacker, unpackb except ImportError: - from .fallback import Packer, unpackb, Unpacker + from .fallback import Packer, Unpacker, unpackb def pack(o, stream, **kwargs): diff --git a/msgpack/ext.py b/msgpack/ext.py index 3940fe0f..9694819a 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,6 +1,6 @@ -from collections import namedtuple import datetime import struct +from collections import namedtuple class ExtType(namedtuple("ExtType", "code data")): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index cbf0d30e..b02e47cf 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,28 +1,22 @@ """Fallback pure Python implementation of msgpack""" -from datetime import datetime as _DateTime -import sys import struct - +import sys +from datetime import datetime as _DateTime if hasattr(sys, "pypy_version_info"): - # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own - # StringBuilder is fastest. from __pypy__ import newlist_hint + from __pypy__.builders import BytesBuilder - try: - from __pypy__.builders import BytesBuilder as StringBuilder - except ImportError: - from __pypy__.builders import StringBuilder - USING_STRINGBUILDER = True + _USING_STRINGBUILDER = True - class StringIO: + class BytesIO: def __init__(self, s=b""): if s: - self.builder = StringBuilder(len(s)) + self.builder = BytesBuilder(len(s)) self.builder.append(s) else: - self.builder = StringBuilder() + self.builder = BytesBuilder() def write(self, s): if isinstance(s, memoryview): @@ -35,17 +29,17 @@ def getvalue(self): return self.builder.build() else: - USING_STRINGBUILDER = False - from io import BytesIO as StringIO + from io import BytesIO - newlist_hint = lambda size: [] + _USING_STRINGBUILDER = False + def newlist_hint(size): + return [] -from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError +from .exceptions import BufferFull, ExtraData, FormatError, OutOfData, StackError from .ext import ExtType, Timestamp - EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 @@ -335,6 +329,7 @@ def feed(self, next_bytes): # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython self._buffer.extend(view) + view.release() def _consume(self): """Gets rid of the used parts of the buffer.""" @@ -671,12 +666,11 @@ def __init__( self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type - self._buffer = StringIO() + self._buffer = BytesIO() self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" - if default is not None: - if not callable(default): - raise TypeError("default must be callable") + if default is not None and not callable(default): + raise TypeError("default must be callable") self._default = default def _pack( @@ -807,18 +801,18 @@ def pack(self, obj): try: self._pack(obj) except: - self._buffer = StringIO() # force reset + self._buffer = BytesIO() # force reset raise if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_map_pairs(self, pairs): self._pack_map_pairs(len(pairs), pairs) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_array_header(self, n): @@ -827,7 +821,7 @@ def pack_array_header(self, n): self._pack_array_header(n) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_map_header(self, n): @@ -836,7 +830,7 @@ def pack_map_header(self, n): self._pack_map_header(n) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_ext_type(self, typecode, data): @@ -925,11 +919,11 @@ def reset(self): This method is useful only when autoreset=False. """ - self._buffer = StringIO() + self._buffer = BytesIO() def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER: + if _USING_STRINGBUILDER: return memoryview(self.bytes()) else: return self._buffer.getbuffer() diff --git a/pyproject.toml b/pyproject.toml index 6254f06f..f36c7f40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,17 +45,12 @@ include-package-data = false [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} -[tool.black] -line-length = 100 -target-version = ["py37"] -skip_string_normalization = true - [tool.ruff] line-length = 100 target-version = "py38" -lint.ignore = [] - -[tool.ruff.lint.per-file-ignores] -"msgpack/__init__.py" = ["F401", "F403"] -"msgpack/fallback.py" = ["E731"] -"test/test_seq.py" = ["E501"] +lint.select = [ + "E", # pycodestyle + "F", # Pyflakes + "I", # isort + #"UP", pyupgrade +] diff --git a/setup.py b/setup.py index dc14a26e..eaca7460 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ #!/usr/bin/env python import os import sys -from setuptools import setup, Extension + +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist - PYPY = hasattr(sys, "pypy_version_info") diff --git a/test/test_buffer.py b/test/test_buffer.py index 2165eb50..2c5a14c5 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,6 +1,6 @@ from pytest import raises -from msgpack import packb, unpackb, Packer +from msgpack import Packer, packb, unpackb def test_unpack_buffer(): diff --git a/test/test_except.py b/test/test_except.py index 8c0a9766..b77ac800 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +import datetime + from pytest import raises -from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData -import datetime +from msgpack import FormatError, OutOfData, StackError, Unpacker, packb, unpackb class DummyException(Exception): diff --git a/test/test_extension.py b/test/test_extension.py index 9e5e6aad..aaf0fd92 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,4 +1,5 @@ import array + import msgpack from msgpack import ExtType diff --git a/test/test_limits.py b/test/test_limits.py index 533bc112..9b92b4d9 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -2,14 +2,14 @@ import pytest from msgpack import ( - packb, - unpackb, - Packer, - Unpacker, ExtType, + Packer, PackOverflowError, PackValueError, + Unpacker, UnpackValueError, + packb, + unpackb, ) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index eff4bca0..0a2a6f53 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from array import array + from msgpack import packb, unpackb diff --git a/test/test_newspec.py b/test/test_newspec.py index a6f4251b..9e2f9be5 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,4 +1,4 @@ -from msgpack import packb, unpackb, ExtType +from msgpack import ExtType, packb, unpackb def test_str8(): diff --git a/test/test_obj.py b/test/test_obj.py index f78bf426..23be06d5 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from pytest import raises + from msgpack import packb, unpackb diff --git a/test/test_pack.py b/test/test_pack.py index 4a0ef403..374d1549 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,12 +1,12 @@ #!/usr/bin/env python +import struct from collections import OrderedDict from io import BytesIO -import struct import pytest -from msgpack import packb, unpackb, Unpacker, Packer +from msgpack import Packer, Unpacker, packb, unpackb def check(data, use_list=False): diff --git a/test/test_read_size.py b/test/test_read_size.py index a7d61fd9..0f6c1b50 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,6 +1,6 @@ """Test Unpacker's read_array_header and read_map_header methods""" -from msgpack import packb, Unpacker, OutOfData +from msgpack import OutOfData, Unpacker, packb UnexpectedTypeException = ValueError diff --git a/test/test_seq.py b/test/test_seq.py index 16d9dde4..8dee4620 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python - +# ruff: noqa: E501 +# ignore line length limit for long comments import io -import msgpack +import msgpack binarydata = bytes(bytearray(range(256))) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6b138aad..0f895d7d 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import io -from msgpack import Unpacker, BufferFull -from msgpack import pack, packb -from msgpack.exceptions import OutOfData + from pytest import raises +from msgpack import BufferFull, Unpacker, pack, packb +from msgpack.exceptions import OutOfData + def test_partialdata(): unpacker = Unpacker() diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 9ffaff25..72776a2c 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,5 +1,6 @@ from collections import namedtuple -from msgpack import packb, unpackb, ExtType + +from msgpack import ExtType, packb, unpackb def test_namedtuple(): diff --git a/test/test_subtype.py b/test/test_subtype.py index 0d1c41af..a911578c 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -from msgpack import packb from collections import namedtuple +from msgpack import packb + class MyList(list): pass diff --git a/test/test_timestamp.py b/test/test_timestamp.py index f9bc8353..831141a1 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,7 @@ -import pytest import datetime + +import pytest + import msgpack from msgpack.ext import Timestamp diff --git a/test/test_unpack.py b/test/test_unpack.py index bf3f960d..b17c3c53 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,12 +1,9 @@ -from io import BytesIO import sys -from msgpack import Unpacker, packb, OutOfData, ExtType -from pytest import raises, mark +from io import BytesIO + +from pytest import mark, raises -try: - from itertools import izip as zip -except ImportError: - pass +from msgpack import ExtType, OutOfData, Unpacker, packb def test_unpack_array_header_from_file(): From 9cea8b6da23ce66f0e78f017c96adcc447deb09a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 May 2024 20:49:23 +0900 Subject: [PATCH 094/105] Release v1.1.0rc1 (#609) --- ChangeLog.rst | 20 +++++++++++++++++++- msgpack/__init__.py | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 2408bc9f..9b16e41f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,21 @@ +1.1.0rc1 +======== + +Release Date: 2024-05-07 + +* Update Cython to 3.0.10 to reduce C warnings and future support for Python 3.13. +* Stop using C++ mode in Cython to reduce compile error on some compilers. +* ``Packer()`` has ``buf_size`` option to specify initial size of + internal buffer to reduce reallocation. +* The default internal buffer size of ``Packer()`` is reduced from + 1MiB to 256KiB to optimize for common use cases. Use ``buf_size`` + if you are packing large data. +* ``Timestamp.to_datetime()`` and ``Timestamp.from_datetime()`` become + more accurate by avoiding floating point calculations. (#591) +* The Cython code for ``Unpacker`` has been slightly rewritten for maintainability. +* The fallback implementation of ``Packer()`` and ``Unpacker()`` now uses keyword-only + arguments to improve compatibility with the Cython implementation. + 1.0.8 ===== @@ -130,7 +148,7 @@ Important changes * unpacker: Default value of input limits are smaller than before to avoid DoS attack. If you need to handle large data, you need to specify limits manually. (#319) -* Unpacker doesn't wrap underlaying ``ValueError`` (including ``UnicodeError``) into +* Unpacker doesn't wrap underlying ``ValueError`` (including ``UnicodeError``) into ``UnpackValueError``. If you want to catch all exception during unpack, you need to use ``try ... except Exception`` with minimum try code block. (#323, #233) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index e796efb0..c2525444 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 0, 8) -__version__ = "1.0.8" +version = (1, 1, 0, "rc1") +__version__ = "1.1.0rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From 0b1c47b06b55d91c00c9f7153c4a9440ea878886 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 May 2024 22:01:54 +0900 Subject: [PATCH 095/105] do not install cython as build dependency (#610) User can not cythonize during `pip install msgpack`. So remove cython from build dependency. If user need to use another Cython, user should download sdist, unzip, manually cythonize, and `pip install .`. --- pyproject.toml | 6 +----- setup.py | 45 --------------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f36c7f40..d041d4c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,5 @@ [build-system] -requires = [ - # Also declared in requirements.txt, if updating here please also update there - "Cython~=3.0.10", - "setuptools >= 69.5.1", -] +requires = ["setuptools >= 69.5.1"] build-backend = "setuptools.build_meta" [project] diff --git a/setup.py b/setup.py index eaca7460..4029e9ed 100644 --- a/setup.py +++ b/setup.py @@ -3,52 +3,9 @@ import sys from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext -from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") - -class NoCython(Exception): - pass - - -try: - import Cython.Compiler.Main as cython_compiler - - have_cython = True -except ImportError: - have_cython = False - - -def cythonize(src): - if not have_cython: - raise Exception("Cython is required for building from checkout") - sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src]) - - -def ensure_source(src): - pyx = os.path.splitext(src)[0] + ".pyx" - - if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime: - cythonize(pyx) - - -class BuildExt(build_ext): - def build_extension(self, ext): - for src in ext.sources: - ensure_source(src) - return build_ext.build_extension(self, ext) - - -# Cython is required for sdist -class Sdist(sdist): - def __init__(self, *args, **kwargs): - cythonize("msgpack/_cmsgpack.pyx") - sdist.__init__(self, *args, **kwargs) - - libraries = [] macros = [] ext_modules = [] @@ -69,9 +26,7 @@ def __init__(self, *args, **kwargs): ) del libraries, macros - setup( - cmdclass={"build_ext": BuildExt, "sdist": Sdist}, ext_modules=ext_modules, packages=["msgpack"], ) From 6e11368f5d54f7d4878dc209717495c37be03c68 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:35:16 +0900 Subject: [PATCH 096/105] update Cython to 3.0.11 (#617) --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1164a941..b677f068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -# Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.10 +Cython~=3.0.11 From 9e26d80ab2a02221d3ca36cc0b5ca2268f391204 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:56:01 +0900 Subject: [PATCH 097/105] update cibuildwheel to 2.20.0 (#618) --- .github/workflows/wheel.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d57e0586..01d0bbd1 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,7 +32,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -40,13 +40,11 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* - - name: Build pure Python wheel + - name: Build sdist if: runner.os == 'Linux' - env: - MSGPACK_PUREPYTHON: "1" run: | pip install build - python -m build -w -o wheelhouse + python -m build -s -o wheelhouse - name: Upload Wheels to artifact uses: actions/upload-artifact@v4 From 9d0c7f2f9cab21c3e39d23001cd9d2034824ee61 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 20:36:26 +0900 Subject: [PATCH 098/105] Release v1.1.0rc2 (#619) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 01d0bbd1..50157223 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -38,7 +38,7 @@ jobs: CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_ARCHS_LINUX: auto aarch64 CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_SKIP: pp* + CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist if: runner.os == 'Linux' diff --git a/ChangeLog.rst b/ChangeLog.rst index 9b16e41f..47328ffb 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0rc2 +======== + +Release Date: 2024-08-19 + +* Update Cython to 3.0.11 for better Python 3.13 support. +* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. + 1.1.0rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index c2525444..a72e974c 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc1") -__version__ = "1.1.0rc1" +version = (1, 1, 0, "rc2") +__version__ = "1.1.0rc2" if os.environ.get("MSGPACK_PUREPYTHON"): From 20a2b8eaa26d76169049ce150e10d1d2aa37d3ab Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 21 Aug 2024 01:56:00 -0400 Subject: [PATCH 099/105] use `PyLong_*` instead of `PyInt_*` (#620) 9af421163cb8081414be347038dee7a82b29e8dd in Cython removed back-compatibility `#define`. --- msgpack/unpack.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 23aa6220..58a2f4f5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -47,7 +47,7 @@ static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong((long)d); + PyObject *p = PyLong_FromLong((long)d); if (!p) return -1; *o = p; @@ -61,7 +61,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromSize_t((size_t)d); + PyObject *p = PyLong_FromSize_t((size_t)d); if (!p) return -1; *o = p; @@ -74,7 +74,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } if (!p) return -1; @@ -84,7 +84,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong(d); + PyObject *p = PyLong_FromLong(d); if (!p) return -1; *o = p; @@ -107,7 +107,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac if (d > LONG_MAX || d < LONG_MIN) { p = PyLong_FromLongLong((PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } *o = p; return 0; From 4587393b1ae2c9ebbd5bc93005b8aea2c8050b27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 10 Sep 2024 01:58:00 +0900 Subject: [PATCH 100/105] release v1.1.0 (#622) --- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 47328ffb..863c6b2c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0 +===== + +Release Date: 2024-09-10 + +* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with + future Cython. (#620) + 1.1.0rc2 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index a72e974c..b6151054 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc2") -__version__ = "1.1.0rc2" +version = (1, 1, 0) +__version__ = "1.1.0" if os.environ.get("MSGPACK_PUREPYTHON"): From 0eeabfb453844b441a4a77097b3d5aa0cb6645b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Tue, 8 Oct 2024 03:04:56 -0600 Subject: [PATCH 101/105] Add Python 3.13 trove classifier (#626) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d041d4c0..e24f2b88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 868aa2cd83f39237deb957c68ce7232422a5950b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 31 May 2025 12:45:06 +0900 Subject: [PATCH 102/105] update Cython to 3.1.1 (#637) --- .github/workflows/test.yml | 9 +++++++-- pyproject.toml | 12 +++++++----- requirements.txt | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 530238c9..23d221c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -26,10 +26,15 @@ jobs: allow-prereleases: true cache: "pip" - - name: Build + - name: Prepare shell: bash run: | + pip install -U pip pip install -r requirements.txt pytest + + - name: Build + shell: bash + run: | make cython pip install . diff --git a/pyproject.toml b/pyproject.toml index e24f2b88..b1628322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ [build-system] -requires = ["setuptools >= 69.5.1"] +# 75.3.0 is the latest version supporting Python 3.8 +requires = ["setuptools >= 75.3.0"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] +# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility. +# https://github.com/msgpack/msgpack-python/pull/637 license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" @@ -14,18 +17,17 @@ requires-python = ">=3.8" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", + "Topic :: File Formats", + "Intended Audience :: Developers", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index b677f068..78a2f38f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.0.11 +Cython~=3.1.1 From cdc764450370ff80e7c83edbe8d015f08f6fb9b3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Jun 2025 16:56:44 +0900 Subject: [PATCH 103/105] update cibuildwheel to v2.23.3 (#638) --- .github/workflows/wheel.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 50157223..d97de1dd 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -9,17 +9,12 @@ jobs: build_wheels: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + # macos-13 is for intel + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} steps: - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -32,12 +27,10 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_ARCHS_LINUX: auto aarch64 - CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist From fe9e620a607702b31476f092ad01a387cff4cfbd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Jun 2025 14:46:53 +0900 Subject: [PATCH 104/105] upload to PyPI on create a release (#639) --- .github/workflows/wheel.yml | 48 ++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d97de1dd..686d7dd0 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -1,8 +1,10 @@ -name: Build Wheels +name: Build sdist and Wheels on: push: branches: [main] - create: + release: + types: + - published workflow_dispatch: jobs: @@ -34,7 +36,7 @@ jobs: CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.arch == 'X64' run: | pip install build python -m build -s -o wheelhouse @@ -44,3 +46,43 @@ jobs: with: name: wheels-${{ matrix.os }} path: wheelhouse + + # combine all wheels into one artifact + combine_wheels: + needs: [build_wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Upload Wheels to artifact + uses: actions/upload-artifact@v4 + with: + name: wheels-all + path: dist + + # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml + upload_pypi: + needs: [build_wheels] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + #with: + # To test: repository-url: https://test.pypi.org/legacy/ From e6445d3b922ca0b9bc82695dd9d1c1529763095a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Jun 2025 09:38:38 +0900 Subject: [PATCH 105/105] v1.1.1rc1 --- ChangeLog.rst | 7 +++++++ docs/conf.py | 6 +++--- msgpack/__init__.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 863c6b2c..0f67502c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1rc1 +======== + +Release Date: 2025-06-06 + +* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + 1.1.0 ===== diff --git a/docs/conf.py b/docs/conf.py index ab0ad3c8..28116cd6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,9 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#import os -#import sys -#sys.path.insert(0, os.path.abspath('..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b6151054..b8d83da8 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0) -__version__ = "1.1.0" +version = (1, 1, 1, "rc1") +__version__ = "1.1.1rc1" if os.environ.get("MSGPACK_PUREPYTHON"):