8000
We read every piece of feedback, and take your input very seriously.
1 parent 98e2c3a commit 1978904Copy full SHA for 1978904
Lib/compression/zstd/__init__.py
@@ -71,7 +71,7 @@ def get_frame_info(frame_buffer):
71
the frame may or may not need a dictionary to be decoded,
72
and the ID of such a dictionary is not specified.
73
"""
74
- return FrameInfo(*_zstd._get_frame_info(frame_buffer))
+ return FrameInfo(*_zstd.get_frame_info(frame_buffer))
75
76
77
def train_dict(samples, dict_size):
@@ -91,7 +91,7 @@ def train_dict(samples, dict_size):
91
chunk_sizes = tuple(_nbytes(sample) for sample in samples)
92
if not chunks:
93
raise ValueError("samples contained no data; can't train dictionary.")
94
- dict_content = _zstd._train_dict(chunks, chunk_sizes, dict_size)
+ dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size)
95
return ZstdDict(dict_content)
96
97
@@ -127,7 +127,7 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level):
127
128
raise ValueError("The samples are empty content, can't finalize the"
129
"dictionary.")
130
- dict_content = _zstd._finalize_dict(zstd_dict.dict_content,
+ dict_content = _zstd.finalize_dict(zstd_dict.dict_content,
131
chunks, chunk_sizes,
132
dict_size, level)
133
@@ -201,7 +201,7 @@ def bounds(self):
201
202
Both the lower and upper bounds are inclusive.
203
204
- return _zstd._get_param_bounds(self.value, is_compress=True)
+ return _zstd.get_param_bounds(self.value, is_compress=True)
205
206
207
class DecompressionParameter(enum.IntEnum):
@@ -214,7 +214,7 @@ def bounds(self):
214
215
216
217
- return _zstd._get_param_bou 67E6 nds(self.value, is_compress=False)
+ return _zstd.get_param_bounds(self.value, is_compress=False)
218
219
220
class Strategy(enum.IntEnum):
@@ -237,4 +237,4 @@ class Strategy(enum.IntEnum):
237
238
239
# Check validity of the CompressionParameter & DecompressionParameter types
240
-_zstd._set_parameter_types(CompressionParameter, DecompressionParameter)
+_zstd.set_parameter_types(CompressionParameter, DecompressionParameter)
Lib/test/test_zstd.py
@@ -1174,43 +1174,43 @@ def test_finalize_dict_arguments(self):
1174
def test_train_dict_c(self):
1175
# argument wrong type
1176
with self.assertRaises(TypeError):
1177
- _zstd._train_dict({}, (), 100)
+ _zstd.train_dict({}, (), 100)
1178
1179
- _zstd._train_dict(b'', 99, 100)
+ _zstd.train_dict(b'', 99, 100)
1180
1181
- _zstd._train_dict(b'', (), 100.1)
+ _zstd.train_dict(b'', (), 100.1)
1182
1183
# size > size_t
1184
with self.assertRaises(ValueError):
1185
- _zstd._train_dict(b'', (2**64+1,), 100)
+ _zstd.train_dict(b'', (2**64+1,), 100)
1186
1187
# dict_size <= 0
1188
1189
- _zstd._train_dict(b'', (), 0)
+ _zstd.train_dict(b'', (), 0)
1190
1191
def test_finalize_dict_c(self):
1192
1193
- _zstd._finalize_dict(1, 2, 3, 4, 5)
+ _zstd.finalize_dict(1, 2, 3, 4, 5)
1194
1195
1196
1197
- _zstd._finalize_dict({}, b'', (), 100, 5)
+ _zstd.finalize_dict({}, b'', (), 100, 5)
1198
1199
- _zstd._finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
+ _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
1200
1201
- _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
+ _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
1202
1203
- _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
+ _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
1204
1205
- _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)
+ _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)
1206
1207
1208
1209
- _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5)
+ _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5)
1210
1211
1212
1213
- _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)
+ _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)
1214
1215
def test_train_buffer_protocol_samples(self):
1216
def _nbytes(dat):
@@ -1232,19 +1232,19 @@ def _nbytes(dat):
1232
# wrong size list
1233
with self.assertRaisesRegex(ValueError,
1234
"The samples size tuple doesn't match the concatenation's size"):
1235
- _zstd._train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)
+ _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)
1236
1237
# correct size list
1238
- _zstd._train_dict(concatenation, tuple(correct_size_lst), 3*_1K)
+ _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K)
1239
1240
1241
1242
1243
- _zstd._finalize_dict(TRAINED_DICT.dict_content,
+ _zstd.finalize_dict(TRAINED_DICT.dict_content,
1244
concatenation, tuple(wrong_size_lst), 300*_1K, 5)
1245
1246
1247
1248
concatenation, tuple(correct_size_lst), 300*_1K, 5)
1249
1250
def test_as_prefix(self):
Modules/_zstd/_zstdmodule.c
@@ -69,8 +69,7 @@ typedef struct {
69
char parameter_name[32];
70
} ParameterInfo;
-static const ParameterInfo cp_list[] =
-{
+static const ParameterInfo cp_list[] = {
{ZSTD_c_compressionLevel, "compression_level"},
{ZSTD_c_windowLog, "window_log"},
{ZSTD_c_hashLog, "hash_log"},
@@ -95,8 +94,7 @@ static const ParameterInfo cp_list[] =
{ZSTD_c_overlapLog, "overlap_log"}
};
98
-static const ParameterInfo dp_list[] =
99
+static const ParameterInfo dp_list[] = {
100
{ZSTD_d_windowLogMax, "window_log_max"}
101
102
@@ -173,7 +171,7 @@ get_zstd_state(PyObject *module)
173
171
174
172
175
/*[clinic input]
176
-_zstd._train_dict
+_zstd.train_dict
177
178
samples_bytes: PyBytesObject
179
Concatenation of samples.
@@ -187,9 +185,9 @@ Internal function, train a zstd dictionary on sample data.
187
185
[clinic start generated code]*/
188
186
189
static PyObject *
190
-_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
191
- PyObject *samples_sizes, Py_ssize_t dict_size)
192
-/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/
+_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
+ PyObject *samples_sizes, Py_ssize_t dict_size)
+/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/
193
{
194
// TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
195
// are pretty similar. We should see if we can refactor them to share that code.
@@ -277,7 +275,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
277
275
}
278
276
279
280
-_zstd._finalize_dict
+_zstd.finalize_dict
281
282
custom_dict_bytes: PyBytesObject
283
Custom dictionary content.
@@ -295,11 +293,11 @@ Internal function, finalize a zstd dictionary.
295
293
296
294
297
298
-_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
299
- PyBytesObject *samples_bytes,
300
- PyObject *samples_sizes, Py_ssize_t dict_size,
301
- int compression_level)
302
-/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/
+_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
+ PyBytesObject *samples_bytes,
+ PyObject *samples_sizes, Py_ssize_t dict_size,
+ int compression_level)
+/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/
303
304
Py_ssize_t chunks_number;
305
size_t *chunk_sizes = NULL;
@@ -396,7 +394,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
396
394
397
395
398
399
-_zstd._get_param_bounds
+_zstd.get_param_bounds
400
401
parameter: int
402
The parameter to get bounds.
@@ -407,9 +405,8 @@ Internal function, get CompressionParameter/DecompressionParameter bounds.
407
405
408
406
409
410
-_zstd__get_param_bounds_impl(PyObject *module, int parameter,
411
- int is_compress)
412
-/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/
+_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
+/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/
413
414
ZSTD_bounds bound;
415
if (is_compress) {
@@ -466,7 +463,7 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
466
463
467
464
468
465
469
-_zstd._get_frame_info
+_zstd.get_frame_info
470
471
frame_buffer: Py_buffer
472
A bytes-like object, containing the header of a zstd frame.
@@ -475,8 +472,8 @@ Internal function, get zstd frame infomation from a frame header.
475
476
473
477
474
478
-_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
479
-/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/
+_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
+/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/
480
481
uint64_t decompressed_size;
482
uint32_t dict_id;
@@ -508,7 +505,7 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
508
505
509
506
510
507
511
-_zstd._set_parameter_types
+_zstd.set_parameter_types
512
513
c_parameter_type: object(subclass_of='&PyType_Type')
514
CompressionParameter IntEnum type object
@@ -519,9 +516,9 @@ Internal function, set CompressionParameter/DecompressionParameter types for val
519
516
520
517
521
518
522
-_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
523
- PyObject *d_parameter_type)
524
-/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/
+_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
+ PyObject *d_parameter_type)
+/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/
525
526
_zstd_state* const mod_state = get_zstd_state(module);
527
@@ -544,14 +541,13 @@ _zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
544
541
545
542
546
543
static PyMethodDef _zstd_methods[] = {
547
- _ZSTD__TRAIN_DICT_METHODDEF
548
- _ZSTD__FINALIZE_DICT_METHODDEF
549
- _ZSTD__GET_PARAM_BOUNDS_METHODDEF
+ _ZSTD_TRAIN_DICT_METHODDEF
+ _ZSTD_FINALIZE_DICT_METHODDEF
+ _ZSTD_GET_PARAM_BOUNDS_METHODDEF
550
_ZSTD_GET_FRAME_SIZE_METHODDEF
551
- _ZSTD__GET_FRAME_INFO_METHODDEF
552
- _ZSTD__SET_PARAMETER_TYPES_METHODDEF
553
-
554
- {0}
+ _ZSTD_GET_FRAME_INFO_METHODDEF
+ _ZSTD_SET_PARAMETER_TYPES_METHODDEF
+ {NULL, NULL}
555
556
557
static int
@@ -595,7 +591,7 @@ do { \
595
591
ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec);
596
592
ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec);
597
593
mod_state->ZstdError = PyErr_NewExceptionWithDoc(
598
- "_zstd.ZstdError",
594
+ "compression.zstd.ZstdError",
599
"An error occurred in the zstd library.",
600
NULL, NULL);
601
if (mod_state->ZstdError == NULL) {
@@ -732,14 +728,15 @@ static struct PyModuleDef_Slot _zstd_slots[] = {
732
728
733
729
734
730
struct PyModuleDef _zstdmodule = {
735
- PyModuleDef_HEAD_INIT,
731
+ .m_base = PyModuleDef_HEAD_INIT,
736
.m_name = "_zstd",
+ .m_doc = "Implementation module for Zstandard compression.",
737
.m_size = sizeof(_zstd_state),
738
.m_slots = _zstd_slots,
739
.m_methods = _zstd_methods,
740
.m_traverse = _zstd_traverse,
741
.m_clear = _zstd_clear,
742
- .m_free = _zstd_free
+ .m_free = _zstd_free,
743
744
745
PyMODINIT_FUNC
Modules/_zstd/_zstdmodule.h
@@ -20,7 +20,8 @@ extern PyModuleDef _zstdmodule;
20
21
/* For clinic type calculations */
22
static inline _zstd_state *
23
-get_zstd_state_from_type(PyTypeObject *type) {
+get_zstd_state_from_type(PyTypeObject *type)
24
+{
25
PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
26
if (module == NULL) {
27
return NULL;
@@ -149,7 +150,8 @@ typedef enum {
149
150
} dictionary_type;
151
152
static inline int
-mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) {
153
+mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
154
155
return in->size == in->pos && out->size != out->pos;
156
157
@@ -163,13 +165,3 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
163
165
int key_v, int value_v);
164
166
167
static const char init_twice_msg[] = "__init__ method is called twice.";
-extern PyObject *
168
-decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
169
- Py_ssize_t max_length,
170
- Py_ssize_t initial_size,
- decompress_type type);
-compress_impl(ZstdCompressor *self, Py_buffer *data,
- ZSTD_EndDirective end_directive);