From 0c3c6670eb89f4369c8e26b0e388687eaebe00d1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:42:18 +0100 Subject: [PATCH 01/15] zstd: replace star import with explicit names --- Lib/compression/zstd/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 4f734eb07b00e3..0aaf8edebb19ab 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -28,7 +28,8 @@ import _zstd import enum -from _zstd import * +from _zstd import (get_frame_size, zstd_version, zstd_version_info, + ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError) from compression.zstd._zstdfile import ZstdFile, open, _nbytes COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0] From e02f66cd87812cc4a60ba52ed1ec691b93a1ef22 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:43:06 +0100 Subject: [PATCH 02/15] zstd: factor common parts of set_zstd_error() --- Modules/_zstd/_zstdmodule.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 4d046859a1540e..a7e34c0a53ae49 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -28,43 +28,44 @@ set_zstd_error(const _zstd_state* const state, switch (type) { case ERR_DECOMPRESS: - msg = "Unable to decompress zstd data: %s"; + msg = "decompress zstd data"; break; case ERR_COMPRESS: - msg = "Unable to compress zstd data: %s"; + msg = "compress zstd data"; break; case ERR_SET_PLEDGED_INPUT_SIZE: - msg = "Unable to set pledged uncompressed content size: %s"; + msg = "set pledged uncompressed content size"; break; case ERR_LOAD_D_DICT: - msg = "Unable to load zstd dictionary or prefix for decompression: %s"; + msg = "load zstd dictionary or prefix for decompression"; break; case ERR_LOAD_C_DICT: - msg = "Unable to load zstd dictionary or prefix for compression: %s"; + msg = "load zstd dictionary or prefix for compression"; break; case ERR_GET_C_BOUNDS: - msg = "Unable to get zstd compression parameter bounds: %s"; + msg = "get zstd compression parameter bounds"; break; case ERR_GET_D_BOUNDS: - msg = "Unable to get zstd decompression parameter bounds: %s"; + msg = "get zstd decompression parameter bounds"; break; case ERR_SET_C_LEVEL: - msg = "Unable to set zstd compression level: %s"; + msg = "set zstd compression level"; break; case ERR_TRAIN_DICT: - msg = "Unable to train zstd dictionary: %s"; + msg = "train zstd dictionary"; break; case ERR_FINALIZE_DICT: - msg = "Unable to finalize zstd dictionary: %s"; + msg = "finalize zstd dictionary"; break; default: Py_UNREACHABLE(); } - PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret)); + PyErr_Format(state->ZstdError, "Unable to %s: %s", + msg, ZSTD_getErrorName(zstd_ret)); } typedef struct { From 686ae618680dcac5d33506912689884db138ffbf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:43:34 +0100 Subject: [PATCH 03/15] zstd: Remove ZSTD_versionString from error messages --- Lib/test/test_zstd.py | 4 ++-- Modules/_zstd/_zstdmodule.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index f4a25376e5234a..cd65c1a66370c8 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -281,7 +281,7 @@ def test_compress_parameters(self): with self.assertRaisesRegex(ZstdError, (r'Error when setting zstd compression parameter "window_log", ' r'it should \d+ <= value <= \d+, provided value is 100\. ' - r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): + r'\((?:32|64)-bit build\)')): compress(b'', options=option) def test_unknown_compression_parameter(self): @@ -413,7 +413,7 @@ def test_decompress_parameters(self): with self.assertRaisesRegex(ZstdError, (r'Error when setting zstd decompression parameter "window_log_max", ' r'it should \d+ <= value <= \d+, provided value is 100\. ' - r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): + r'\((?:32|64)-bit build\)')): decompress(b'', options=options) def test_unknown_decompression_parameter(self): diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index a7e34c0a53ae49..ab8382760869bc 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -152,8 +152,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress, } if (ZSTD_isError(bounds.error)) { PyErr_Format(state->ZstdError, - "Zstd %s parameter \"%s\" is invalid. (zstd v%s)", - type, name, ZSTD_versionString()); + "Zstd %s parameter \"%s\" is invalid.", + type, name); return; } @@ -161,10 +161,10 @@ set_parameter_error(const _zstd_state* const state, int is_compress, PyErr_Format(state->ZstdError, "Error when setting zstd %s parameter \"%s\", it " "should %d <= value <= %d, provided value is %d. " - "(zstd v%s, %d-bit build)", + "(%d-bit build)", type, name, bounds.lowerBound, bounds.upperBound, value_v, - ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t)); + 8*(int)sizeof(Py_ssize_t)); } static inline _zstd_state* From 7ea1ac158ec0b05d7b57b5bc4367d783e7651931 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:45:44 +0100 Subject: [PATCH 04/15] zstd: Simplify version_info setup --- Modules/_zstd/_zstdmodule.c | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index ab8382760869bc..a7e1158862a2cd 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -612,37 +612,25 @@ add_parameters(PyObject *module) return 0; } -static inline PyObject * -get_zstd_version_info(void) -{ - uint32_t ver = ZSTD_versionNumber(); - uint32_t major, minor, release; - - major = ver / 10000; - minor = (ver / 100) % 100; - release = ver % 100; - - return Py_BuildValue("III", major, minor, release); -} - static inline int add_vars_to_module(PyObject *module) { PyObject *obj; - /* zstd_version, a str. */ + /* zstd_version, str */ if (PyModule_AddStringConstant(module, "zstd_version", ZSTD_versionString()) < 0) { return -1; } - /* zstd_version_info, a tuple. */ - obj = get_zstd_version_info(); - if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) { - Py_XDECREF(obj); + /* zstd_version_info, tuple of (int, int, int) */ + PyObject *vi = Py_BuildValue("BBB", ZSTD_VERSION_MAJOR, ZSTD_VERSION_MINOR, + ZSTD_VERSION_RELEASE); + if (PyModule_AddObjectRef(module, "zstd_version_info", vi) < 0) { + Py_XDECREF(vi); return -1; } - Py_DECREF(obj); + Py_DECREF(vi); /* Add zstd parameters */ if (add_parameters(module) < 0) { From 977433844a931d0390032fb1bb380e0d15a50f1a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:46:41 +0100 Subject: [PATCH 05/15] zstd: Replace _compressionLevel_values with ZSTD_CLEVEL_DEFAULT --- Lib/compression/zstd/__init__.py | 2 +- Modules/_zstd/_zstdmodule.c | 26 ++++++++++---------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 0aaf8edebb19ab..f703fa302becd5 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -32,7 +32,7 @@ ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError) from compression.zstd._zstdfile import ZstdFile, open, _nbytes -COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0] +COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT """The default compression level for Zstandard, currently '3'.""" diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index a7e1158862a2cd..a617f24b2b51d1 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -632,28 +632,22 @@ add_vars_to_module(PyObject *module) } Py_DECREF(vi); - /* Add zstd parameters */ - if (add_parameters(module) < 0) { + /* ZSTD_CLEVEL_DEFAULT, int */ +#if ZSTD_VERSION_NUMBER >= 10500 + if (PyModule_AddIntConstant(module, "ZSTD_CLEVEL_DEFAULT", + ZSTD_defaultCLevel()) < 0) { return -1; } - - /* _compressionLevel_values: (default, min, max) - ZSTD_defaultCLevel() was added in zstd v1.5.0 */ - obj = Py_BuildValue("iii", -#if ZSTD_VERSION_NUMBER < 10500 - ZSTD_CLEVEL_DEFAULT, #else - ZSTD_defaultCLevel(), + if (PyModule_AddIntMacro(module, ZSTD_CLEVEL_DEFAULT) < 0) { + return -1; + } #endif - ZSTD_minCLevel(), - ZSTD_maxCLevel()); - if (PyModule_AddObjectRef(module, - "_compressionLevel_values", - obj) < 0) { - Py_XDECREF(obj); + + /* Add zstd parameters */ + if (add_parameters(module) < 0) { return -1; } - Py_DECREF(obj); /* _ZSTD_CStreamSizes */ obj = Py_BuildValue("II", From d1dea984ddcee752876d10489710b04be036f7e6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:48:20 +0100 Subject: [PATCH 06/15] zstd: Remove _ZSTD_CStreamSizes, replace _ZSTD_DStreamSizes with ZSTD_DStreamOutSize --- Lib/compression/zstd/_zstdfile.py | 8 +++----- Modules/_zstd/_zstdmodule.c | 24 +++++------------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index fbc9e02a733626..1ff249965692ae 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -1,13 +1,11 @@ import io from os import PathLike -from _zstd import (ZstdCompressor, ZstdDecompressor, _ZSTD_DStreamSizes, - ZstdError) +from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdError, + ZSTD_DStreamOutSize) from compression._common import _streams __all__ = ("ZstdFile", "open") -_ZSTD_DStreamOutSize = _ZSTD_DStreamSizes[1] - _MODE_CLOSED = 0 _MODE_READ = 1 _MODE_WRITE = 2 @@ -188,7 +186,7 @@ def read1(self, size=-1): # Note this should *not* be io.DEFAULT_BUFFER_SIZE. # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing # a full block is read. - size = _ZSTD_DStreamOutSize + size = ZSTD_DStreamOutSize return self._buffer.read1(size) def readinto(self, b): diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index a617f24b2b51d1..56231b3feb51bb 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -644,30 +644,16 @@ add_vars_to_module(PyObject *module) } #endif - /* Add zstd parameters */ - if (add_parameters(module) < 0) { + /* ZSTD_DStreamOutSize, int */ + if (PyModule_AddIntConstant(module, "ZSTD_DStreamOutSize", + (uint32_t)ZSTD_DStreamOutSize()) < 0) { return -1; } - /* _ZSTD_CStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_CStreamInSize(), - (uint32_t)ZSTD_CStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_DStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_DStreamInSize(), - (uint32_t)ZSTD_DStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) { - Py_XDECREF(obj); + /* Add zstd parameters */ + if (add_parameters(module) < 0) { return -1; } - Py_DECREF(obj); /* _ZSTD_CONFIG */ obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c", From ac1e39cf51cff798ce5e78b7e41506bb86c3451a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:48:37 +0100 Subject: [PATCH 07/15] zstd: Remove _ZSTD_CONFIG --- Modules/_zstd/_zstdmodule.c | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 56231b3feb51bb..91e906e6b6c322 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -615,8 +615,6 @@ add_parameters(PyObject *module) static inline int add_vars_to_module(PyObject *module) { - PyObject *obj; - /* zstd_version, str */ if (PyModule_AddStringConstant(module, "zstd_version", ZSTD_versionString()) < 0) { @@ -655,23 +653,6 @@ add_vars_to_module(PyObject *module) return -1; } - /* _ZSTD_CONFIG */ - obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c", - Py_False, - Py_True, -/* User mremap output buffer */ -#if defined(HAVE_MREMAP) - Py_True -#else - Py_False -#endif - ); - if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - return 0; } From 1a54bf3947c3f89673d954970854298593678940 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 8 May 2025 14:54:35 +0100 Subject: [PATCH 08/15] zstd: Replace add_parameters and ADD_INT_PREFIX_MACRO with PyModule_AddIntMacro --- Lib/compression/zstd/__init__.py | 64 +++++++++++----------- Modules/_zstd/_zstdmodule.c | 94 +++++++++++++------------------- 2 files changed, 69 insertions(+), 89 deletions(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index f703fa302becd5..ddf69233b51130 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -168,28 +168,28 @@ def decompress(data, zstd_dict=None, options=None): class CompressionParameter(enum.IntEnum): """Compression parameters.""" - compression_level = _zstd._ZSTD_c_compressionLevel - window_log = _zstd._ZSTD_c_windowLog - hash_log = _zstd._ZSTD_c_hashLog - chain_log = _zstd._ZSTD_c_chainLog - search_log = _zstd._ZSTD_c_searchLog - min_match = _zstd._ZSTD_c_minMatch - target_length = _zstd._ZSTD_c_targetLength - strategy = _zstd._ZSTD_c_strategy - - enable_long_distance_matching = _zstd._ZSTD_c_enableLongDistanceMatching - ldm_hash_log = _zstd._ZSTD_c_ldmHashLog - ldm_min_match = _zstd._ZSTD_c_ldmMinMatch - ldm_bucket_size_log = _zstd._ZSTD_c_ldmBucketSizeLog - ldm_hash_rate_log = _zstd._ZSTD_c_ldmHashRateLog - - content_size_flag = _zstd._ZSTD_c_contentSizeFlag - checksum_flag = _zstd._ZSTD_c_checksumFlag - dict_id_flag = _zstd._ZSTD_c_dictIDFlag - - nb_workers = _zstd._ZSTD_c_nbWorkers - job_size = _zstd._ZSTD_c_jobSize - overlap_log = _zstd._ZSTD_c_overlapLog + compression_level = _zstd.ZSTD_c_compressionLevel + window_log = _zstd.ZSTD_c_windowLog + hash_log = _zstd.ZSTD_c_hashLog + chain_log = _zstd.ZSTD_c_chainLog + search_log = _zstd.ZSTD_c_searchLog + min_match = _zstd.ZSTD_c_minMatch + target_length = _zstd.ZSTD_c_targetLength + strategy = _zstd.ZSTD_c_strategy + + enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching + ldm_hash_log = _zstd.ZSTD_c_ldmHashLog + ldm_min_match = _zstd.ZSTD_c_ldmMinMatch + ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog + ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog + + content_size_flag = _zstd.ZSTD_c_contentSizeFlag + checksum_flag = _zstd.ZSTD_c_checksumFlag + dict_id_flag = _zstd.ZSTD_c_dictIDFlag + + nb_workers = _zstd.ZSTD_c_nbWorkers + job_size = _zstd.ZSTD_c_jobSize + overlap_log = _zstd.ZSTD_c_overlapLog def bounds(self): """Return the (lower, upper) int bounds of a compression parameter. @@ -202,7 +202,7 @@ def bounds(self): class DecompressionParameter(enum.IntEnum): """Decompression parameters.""" - window_log_max = _zstd._ZSTD_d_windowLogMax + window_log_max = _zstd.ZSTD_d_windowLogMax def bounds(self): """Return the (lower, upper) int bounds of a decompression parameter. @@ -220,15 +220,15 @@ class Strategy(enum.IntEnum): the numeric value might change. """ - fast = _zstd._ZSTD_fast - dfast = _zstd._ZSTD_dfast - greedy = _zstd._ZSTD_greedy - lazy = _zstd._ZSTD_lazy - lazy2 = _zstd._ZSTD_lazy2 - btlazy2 = _zstd._ZSTD_btlazy2 - btopt = _zstd._ZSTD_btopt - btultra = _zstd._ZSTD_btultra - btultra2 = _zstd._ZSTD_btultra2 + fast = _zstd.ZSTD_fast + dfast = _zstd.ZSTD_dfast + greedy = _zstd.ZSTD_greedy + lazy = _zstd.ZSTD_lazy + lazy2 = _zstd.ZSTD_lazy2 + btlazy2 = _zstd.ZSTD_btlazy2 + btopt = _zstd.ZSTD_btopt + btultra = _zstd.ZSTD_btultra + btultra2 = _zstd.ZSTD_btultra2 # Check validity of the CompressionParameter & DecompressionParameter types diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 91e906e6b6c322..8857840185d30d 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -559,59 +559,6 @@ static PyMethodDef _zstd_methods[] = { }; -#define ADD_INT_PREFIX_MACRO(module, macro) \ - do { \ - if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \ - return -1; \ - } \ - } while(0) - -static int -add_parameters(PyObject *module) -{ - /* If add new parameters, please also add to cp_list/dp_list above. */ - - /* Compression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog); - - /* Decompression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax); - - /* ZSTD_strategy enum */ - ADD_INT_PREFIX_MACRO(module, ZSTD_fast); - ADD_INT_PREFIX_MACRO(module, ZSTD_dfast); - ADD_INT_PREFIX_MACRO(module, ZSTD_greedy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btopt); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2); - - return 0; -} - static inline int add_vars_to_module(PyObject *module) { @@ -648,10 +595,43 @@ add_vars_to_module(PyObject *module) return -1; } - /* Add zstd parameters */ - if (add_parameters(module) < 0) { - return -1; - } + /* Add zstd compression parameters. All should also be in cp_list. */ + PyModule_AddIntMacro(module, ZSTD_c_compressionLevel); + PyModule_AddIntMacro(module, ZSTD_c_windowLog); + PyModule_AddIntMacro(module, ZSTD_c_hashLog); + PyModule_AddIntMacro(module, ZSTD_c_chainLog); + PyModule_AddIntMacro(module, ZSTD_c_searchLog); + PyModule_AddIntMacro(module, ZSTD_c_minMatch); + PyModule_AddIntMacro(module, ZSTD_c_targetLength); + PyModule_AddIntMacro(module, ZSTD_c_strategy); + + PyModule_AddIntMacro(module, ZSTD_c_enableLongDistanceMatching); + PyModule_AddIntMacro(module, ZSTD_c_ldmHashLog); + PyModule_AddIntMacro(module, ZSTD_c_ldmMinMatch); + PyModule_AddIntMacro(module, ZSTD_c_ldmBucketSizeLog); + PyModule_AddIntMacro(module, ZSTD_c_ldmHashRateLog); + + PyModule_AddIntMacro(module, ZSTD_c_contentSizeFlag); + PyModule_AddIntMacro(module, ZSTD_c_checksumFlag); + PyModule_AddIntMacro(module, ZSTD_c_dictIDFlag); + + PyModule_AddIntMacro(module, ZSTD_c_nbWorkers); + PyModule_AddIntMacro(module, ZSTD_c_jobSize); + PyModule_AddIntMacro(module, ZSTD_c_overlapLog); + + /* Add zstd decompression parameters. All should also be in dp_list. */ + PyModule_AddIntMacro(module, ZSTD_d_windowLogMax); + + /* ZSTD_strategy enum */ + PyModule_AddIntMacro(module, ZSTD_fast); + PyModule_AddIntMacro(module, ZSTD_dfast); + PyModule_AddIntMacro(module, ZSTD_greedy); + PyModule_AddIntMacro(module, ZSTD_lazy); + PyModule_AddIntMacro(module, ZSTD_lazy2); + PyModule_AddIntMacro(module, ZSTD_btlazy2); + PyModule_AddIntMacro(module, ZSTD_btopt); + PyModule_AddIntMacro(module, ZSTD_btultra); + PyModule_AddIntMacro(module, ZSTD_btultra2); return 0; } From 663e6666905d20c15b39fa338e036851485fe50b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 00:40:26 +0100 Subject: [PATCH 09/15] Revert "zstd: factor common parts of set_zstd_error()" This reverts commit e02f66cd87812cc4a60ba52ed1ec691b93a1ef22. --- Modules/_zstd/_zstdmodule.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 8857840185d30d..1b9bfb61b28207 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -28,44 +28,43 @@ set_zstd_error(const _zstd_state* const state, switch (type) { case ERR_DECOMPRESS: - msg = "decompress zstd data"; + msg = "Unable to decompress zstd data: %s"; break; case ERR_COMPRESS: - msg = "compress zstd data"; + msg = "Unable to compress zstd data: %s"; break; case ERR_SET_PLEDGED_INPUT_SIZE: - msg = "set pledged uncompressed content size"; + msg = "Unable to set pledged uncompressed content size: %s"; break; case ERR_LOAD_D_DICT: - msg = "load zstd dictionary or prefix for decompression"; + msg = "Unable to load zstd dictionary or prefix for decompression: %s"; break; case ERR_LOAD_C_DICT: - msg = "load zstd dictionary or prefix for compression"; + msg = "Unable to load zstd dictionary or prefix for compression: %s"; break; case ERR_GET_C_BOUNDS: - msg = "get zstd compression parameter bounds"; + msg = "Unable to get zstd compression parameter bounds: %s"; break; case ERR_GET_D_BOUNDS: - msg = "get zstd decompression parameter bounds"; + msg = "Unable to get zstd decompression parameter bounds: %s"; break; case ERR_SET_C_LEVEL: - msg = "set zstd compression level"; + msg = "Unable to set zstd compression level: %s"; break; case ERR_TRAIN_DICT: - msg = "train zstd dictionary"; + msg = "Unable to train zstd dictionary: %s"; break; case ERR_FINALIZE_DICT: - msg = "finalize zstd dictionary"; + msg = "Unable to finalize zstd dictionary: %s"; break; default: Py_UNREACHABLE(); } - PyErr_Format(state->ZstdError, "Unable to %s: %s", - msg, ZSTD_getErrorName(zstd_ret)); + PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret)); } typedef struct { From dd1380dfa6264eb918af222e48e5c5d83165bcaf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 11:54:01 +0100 Subject: [PATCH 10/15] Use PyModule_Add --- Modules/_zstd/_zstdmodule.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 1b9bfb61b28207..1b2f1af092dc9a 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -568,13 +568,11 @@ add_vars_to_module(PyObject *module) } /* zstd_version_info, tuple of (int, int, int) */ - PyObject *vi = Py_BuildValue("BBB", ZSTD_VERSION_MAJOR, ZSTD_VERSION_MINOR, - ZSTD_VERSION_RELEASE); - if (PyModule_AddObjectRef(module, "zstd_version_info", vi) < 0) { - Py_XDECREF(vi); + if (PyModule_Add(module, "zstd_version_info", + Py_BuildValue("BBB", ZSTD_VERSION_MAJOR, ZSTD_VERSION_MINOR, + ZSTD_VERSION_RELEASE)) < 0) { return -1; } - Py_DECREF(vi); /* ZSTD_CLEVEL_DEFAULT, int */ #if ZSTD_VERSION_NUMBER >= 10500 From fcd65616bdb41d6c9a45e089b015c77817f88639 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 12:12:43 +0100 Subject: [PATCH 11/15] Check failure cases for int macros --- Modules/_zstd/_zstdmodule.c | 85 ++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 1b2f1af092dc9a..938673b74cf0d6 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -559,16 +559,21 @@ static PyMethodDef _zstd_methods[] = { static inline int -add_vars_to_module(PyObject *module) +add_vars_to_module(PyObject *m) { +#define ADD_INT_MACRO(MACRO) \ + if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \ + return -1; \ + } + /* zstd_version, str */ - if (PyModule_AddStringConstant(module, "zstd_version", + if (PyModule_AddStringConstant(m, "zstd_version", ZSTD_versionString()) < 0) { return -1; } /* zstd_version_info, tuple of (int, int, int) */ - if (PyModule_Add(module, "zstd_version_info", + if (PyModule_Add(m, "zstd_version_info", Py_BuildValue("BBB", ZSTD_VERSION_MAJOR, ZSTD_VERSION_MINOR, ZSTD_VERSION_RELEASE)) < 0) { return -1; @@ -576,59 +581,59 @@ add_vars_to_module(PyObject *module) /* ZSTD_CLEVEL_DEFAULT, int */ #if ZSTD_VERSION_NUMBER >= 10500 - if (PyModule_AddIntConstant(module, "ZSTD_CLEVEL_DEFAULT", + if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT", ZSTD_defaultCLevel()) < 0) { return -1; } #else - if (PyModule_AddIntMacro(module, ZSTD_CLEVEL_DEFAULT) < 0) { - return -1; - } + ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT); #endif /* ZSTD_DStreamOutSize, int */ - if (PyModule_AddIntConstant(module, "ZSTD_DStreamOutSize", + if (PyModule_AddIntConstant(m, "ZSTD_DStreamOutSize", (uint32_t)ZSTD_DStreamOutSize()) < 0) { return -1; } /* Add zstd compression parameters. All should also be in cp_list. */ - PyModule_AddIntMacro(module, ZSTD_c_compressionLevel); - PyModule_AddIntMacro(module, ZSTD_c_windowLog); - PyModule_AddIntMacro(module, ZSTD_c_hashLog); - PyModule_AddIntMacro(module, ZSTD_c_chainLog); - PyModule_AddIntMacro(module, ZSTD_c_searchLog); - PyModule_AddIntMacro(module, ZSTD_c_minMatch); - PyModule_AddIntMacro(module, ZSTD_c_targetLength); - PyModule_AddIntMacro(module, ZSTD_c_strategy); - - PyModule_AddIntMacro(module, ZSTD_c_enableLongDistanceMatching); - PyModule_AddIntMacro(module, ZSTD_c_ldmHashLog); - PyModule_AddIntMacro(module, ZSTD_c_ldmMinMatch); - PyModule_AddIntMacro(module, ZSTD_c_ldmBucketSizeLog); - PyModule_AddIntMacro(module, ZSTD_c_ldmHashRateLog); - - PyModule_AddIntMacro(module, ZSTD_c_contentSizeFlag); - PyModule_AddIntMacro(module, ZSTD_c_checksumFlag); - PyModule_AddIntMacro(module, ZSTD_c_dictIDFlag); - - PyModule_AddIntMacro(module, ZSTD_c_nbWorkers); - PyModule_AddIntMacro(module, ZSTD_c_jobSize); - PyModule_AddIntMacro(module, ZSTD_c_overlapLog); + ADD_INT_MACRO(ZSTD_c_compressionLevel); + ADD_INT_MACRO(ZSTD_c_windowLog); + ADD_INT_MACRO(ZSTD_c_hashLog); + ADD_INT_MACRO(ZSTD_c_chainLog); + ADD_INT_MACRO(ZSTD_c_searchLog); + ADD_INT_MACRO(ZSTD_c_minMatch); + ADD_INT_MACRO(ZSTD_c_targetLength); + ADD_INT_MACRO(ZSTD_c_strategy); + + ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching); + ADD_INT_MACRO(ZSTD_c_ldmHashLog); + ADD_INT_MACRO(ZSTD_c_ldmMinMatch); + ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog); + ADD_INT_MACRO(ZSTD_c_ldmHashRateLog); + + ADD_INT_MACRO(ZSTD_c_contentSizeFlag); + ADD_INT_MACRO(ZSTD_c_checksumFlag); + ADD_INT_MACRO(ZSTD_c_dictIDFlag); + + ADD_INT_MACRO(ZSTD_c_nbWorkers); + ADD_INT_MACRO(ZSTD_c_jobSize); + ADD_INT_MACRO(ZSTD_c_overlapLog); /* Add zstd decompression parameters. All should also be in dp_list. */ - PyModule_AddIntMacro(module, ZSTD_d_windowLogMax); + ADD_INT_MACRO(ZSTD_d_windowLogMax); /* ZSTD_strategy enum */ - PyModule_AddIntMacro(module, ZSTD_fast); - PyModule_AddIntMacro(module, ZSTD_dfast); - PyModule_AddIntMacro(module, ZSTD_greedy); - PyModule_AddIntMacro(module, ZSTD_lazy); - PyModule_AddIntMacro(module, ZSTD_lazy2); - PyModule_AddIntMacro(module, ZSTD_btlazy2); - PyModule_AddIntMacro(module, ZSTD_btopt); - PyModule_AddIntMacro(module, ZSTD_btultra); - PyModule_AddIntMacro(module, ZSTD_btultra2); + ADD_INT_MACRO(ZSTD_fast); + ADD_INT_MACRO(ZSTD_dfast); + ADD_INT_MACRO(ZSTD_greedy); + ADD_INT_MACRO(ZSTD_lazy); + ADD_INT_MACRO(ZSTD_lazy2); + ADD_INT_MACRO(ZSTD_btlazy2); + ADD_INT_MACRO(ZSTD_btopt); + ADD_INT_MACRO(ZSTD_btultra); + ADD_INT_MACRO(ZSTD_btultra2); + +#undef ADD_INT_MACRO return 0; } From 06a472375525cc1cb526f3be2f7b8d550f5d631f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 12:15:50 +0100 Subject: [PATCH 12/15] Remove ERR_SET_PLEDGED_INPUT_SIZE --- Modules/_zstd/_zstdmodule.c | 3 --- Modules/_zstd/_zstdmodule.h | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 938673b74cf0d6..ab188fc9f456e8 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -33,9 +33,6 @@ set_zstd_error(const _zstd_state* const state, case ERR_COMPRESS: msg = "Unable to compress zstd data: %s"; break; - case ERR_SET_PLEDGED_INPUT_SIZE: - msg = "Unable to set pledged uncompressed content size: %s"; - break; case ERR_LOAD_D_DICT: msg = "Unable to load zstd dictionary or prefix for decompression: %s"; diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 1e9e4c75056831..0833591c50fcd9 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -137,7 +137,6 @@ typedef enum { typedef enum { ERR_DECOMPRESS, ERR_COMPRESS, - ERR_SET_PLEDGED_INPUT_SIZE, ERR_LOAD_D_DICT, ERR_LOAD_C_DICT, @@ -147,7 +146,7 @@ typedef enum { ERR_SET_C_LEVEL, ERR_TRAIN_DICT, - ERR_FINALIZE_DICT + ERR_FINALIZE_DICT, } error_type; typedef enum { From ef2634d97daef0e3fc861b478c61c7e5d1f04108 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 12:37:26 +0100 Subject: [PATCH 13/15] Use PyLong_FromSize_t --- Modules/_zstd/_zstdmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index ab188fc9f456e8..fa993310f3cf5f 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -587,8 +587,8 @@ add_vars_to_module(PyObject *m) #endif /* ZSTD_DStreamOutSize, int */ - if (PyModule_AddIntConstant(m, "ZSTD_DStreamOutSize", - (uint32_t)ZSTD_DStreamOutSize()) < 0) { + if (PyModule_Add(m, "ZSTD_DStreamOutSize", + PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) { return -1; } From ff28bf241c488513acc5137ec8778affcdf345fd Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 13:51:23 +0100 Subject: [PATCH 14/15] Use runtime version --- Modules/_zstd/_zstdmodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index fa993310f3cf5f..c0be89dc5af783 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -570,9 +570,12 @@ add_vars_to_module(PyObject *m) } /* zstd_version_info, tuple of (int, int, int) */ + const unsigned version = ZSTD_versionNumber(); + const uint8_t v_major = version / 100 / 100; + const uint8_t v_minor = (version / 100) % 100; + const uint8_t v_release = version % 100; if (PyModule_Add(m, "zstd_version_info", - Py_BuildValue("BBB", ZSTD_VERSION_MAJOR, ZSTD_VERSION_MINOR, - ZSTD_VERSION_RELEASE)) < 0) { + Py_BuildValue("BBB", v_major, v_minor, v_release)) < 0) { return -1; } From 3d3dbf7e0cfafa10973ace36d570d7e4eccf31f8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 14:30:28 +0100 Subject: [PATCH 15/15] Calculate zstd_version_info in Python --- Lib/compression/zstd/__init__.py | 9 +++++++-- Modules/_zstd/_zstdmodule.c | 16 ++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index ddf69233b51130..a1054adb64170b 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -28,10 +28,15 @@ import _zstd import enum -from _zstd import (get_frame_size, zstd_version, zstd_version_info, - ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError) +from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, + get_frame_size, zstd_version) from compression.zstd._zstdfile import ZstdFile, open, _nbytes +# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE) +zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100), + _zstd.zstd_version_number % 100) +"""Version number of the runtime zstd library as a tuple of integers.""" + COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT """The default compression level for Zstandard, currently '3'.""" diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index c0be89dc5af783..be900a4c42b54d 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -563,19 +563,15 @@ add_vars_to_module(PyObject *m) return -1; \ } - /* zstd_version, str */ - if (PyModule_AddStringConstant(m, "zstd_version", - ZSTD_versionString()) < 0) { + /* zstd_version_number, int */ + if (PyModule_AddIntConstant(m, "zstd_version_number", + ZSTD_versionNumber()) < 0) { return -1; } - /* zstd_version_info, tuple of (int, int, int) */ - const unsigned version = ZSTD_versionNumber(); - const uint8_t v_major = version / 100 / 100; - const uint8_t v_minor = (version / 100) % 100; - const uint8_t v_release = version % 100; - if (PyModule_Add(m, "zstd_version_info", - Py_BuildValue("BBB", v_major, v_minor, v_release)) < 0) { + /* zstd_version, str */ + if (PyModule_AddStringConstant(m, "zstd_version", + ZSTD_versionString()) < 0) { return -1; }