From 3db20d5b6f1c0b1f42fb81e945fa3a954c8653a4 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 15:14:08 +0100 Subject: [PATCH 01/10] bpo-40479: Fix hashlib's usedforsecurity for OpenSSL 3.0.0 --- .../2022-01-07-15-20-19.bpo-40479.EKfr3F.rst | 2 + Modules/_hashopenssl.c | 482 +++++++++++------- 2 files changed, 306 insertions(+), 178 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-07-15-20-19.bpo-40479.EKfr3F.rst diff --git a/Misc/NEWS.d/next/Library/2022-01-07-15-20-19.bpo-40479.EKfr3F.rst b/Misc/NEWS.d/next/Library/2022-01-07-15-20-19.bpo-40479.EKfr3F.rst new file mode 100644 index 00000000000000..af72923bbd7595 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-07-15-20-19.bpo-40479.EKfr3F.rst @@ -0,0 +1,2 @@ +Fix :mod:`hashlib` *usedforsecurity* option to work correctly with OpenSSL +3.0.0 in FIPS mode. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 12491917832b69..30bb5bdd6b43e4 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -25,6 +25,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_hashtable.h" #include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() @@ -49,6 +50,176 @@ #define PY_OPENSSL_HAS_SHAKE 1 #define PY_OPENSSL_HAS_BLAKE2 1 +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#define PY_EVP_MD EVP_MD +#define PY_EVP_MD_fetch(algorithm, properties) EVP_MD_fetch(NULL, algorithm, properties) +#define PY_EVP_MD_up_ref(md) EVP_MD_up_ref(md) +#define PY_EVP_MD_free(md) EVP_MD_free(md) +#else +#define PY_EVP_MD const EVP_MD +#define PY_EVP_MD_fetch(algorithm, properties) EVP_get_digestbyname(algorithm) +#define PY_EVP_MD_up_ref(md) do {} while(0) +#define PY_EVP_MD_free(md) do {} while(0) +#endif + +/* hash alias map and fast lookup + * + * Map between Python's preferred names and OpenSSL internal names. Maintain + * cache of fetched EVP MD objects. The EVP_get_digestbyname() and + * EVP_MD_fetch() API calls have a performance impact. + * + * The py_hashentry_t items are stored in a _Py_hashtable_t with py_name and + * py_alias as keys. + */ + +enum Py_hash_type { + Py_ht_evp, // usedforsecurity=True / default + Py_ht_evp_nosecurity, // usedforsecurity=False + Py_ht_mac, // HMAC + Py_ht_pbkdf2, // PKBDF2 +}; + +typedef struct { + const char *py_name; + const char *py_alias; + const char *ossl_name; + int ossl_nid; + Py_ssize_t refcnt; + PY_EVP_MD *evp; + PY_EVP_MD *evp_nosecurity; +} py_hashentry_t; + +#define Py_hash_md5 "md5" +#define Py_hash_sha1 "sha1" +#define Py_hash_sha224 "sha224" +#define Py_hash_sha256 "sha256" +#define Py_hash_sha384 "sha384" +#define Py_hash_sha512 "sha512" +#ifdef NID_sha512_224 +#define Py_hash_sha512_224 "sha512_224" +#define Py_hash_sha512_256 "sha512_256" +#endif +#ifdef PY_OPENSSL_HAS_SHA3 +#define Py_hash_sha3_224 "sha3_224" +#define Py_hash_sha3_256 "sha3_256" +#define Py_hash_sha3_384 "sha3_384" +#define Py_hash_sha3_512 "sha3_512" +#endif +#ifdef PY_OPENSSL_HAS_SHAKE +#define Py_hash_shake_128 "shake_128" +#define Py_hash_shake_256 "shake_256" +#endif +#ifdef PY_OPENSSL_HAS_BLAKE2 +#define Py_hash_blake2s "blake2s" +#define Py_hash_blake2b "blake2b" +#endif + +#define PY_HASH_ENTRY(py_name, py_alias, ossl_name, ossl_nid) \ + {py_name, py_alias, ossl_name, ossl_nid, 0, NULL} + +static const py_hashentry_t py_hashes[] = { + /* md5 */ + PY_HASH_ENTRY(Py_hash_md5, "MD5", SN_md5, NID_md5), + /* sha1 */ + PY_HASH_ENTRY(Py_hash_sha1, "SHA1", SN_sha1, NID_sha1), + /* sha2 family */ + PY_HASH_ENTRY(Py_hash_sha224, "SHA224", SN_sha224, NID_sha224), + PY_HASH_ENTRY(Py_hash_sha256, "SHA256", SN_sha256, NID_sha256), + PY_HASH_ENTRY(Py_hash_sha384, "SHA384", SN_sha384, NID_sha384), + PY_HASH_ENTRY(Py_hash_sha512, "SHA512", SN_sha512, NID_sha512), +#ifdef NID_sha512_224 + /* truncated sha2 */ + PY_HASH_ENTRY(Py_hash_sha512_224, "SHA512_224", SN_sha512_224, NID_sha512_224), + PY_HASH_ENTRY(Py_hash_sha512_256, "SHA512_256", SN_sha512_256, NID_sha512_256), +#endif +#ifdef PY_OPENSSL_HAS_SHA3 + /* sha3 */ + PY_HASH_ENTRY(Py_hash_sha3_224, NULL, SN_sha3_224, NID_sha3_224), + PY_HASH_ENTRY(Py_hash_sha3_256, NULL, SN_sha3_256, NID_sha3_256), + PY_HASH_ENTRY(Py_hash_sha3_384, NULL, SN_sha3_384, NID_sha3_384), + PY_HASH_ENTRY(Py_hash_sha3_512, NULL, SN_sha3_512, NID_sha3_512), +#endif +#ifdef PY_OPENSSL_HAS_SHAKE + /* sha3 shake */ + PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), + PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), +#endif +#ifdef PY_OPENSSL_HAS_BLAKE2 + /* blake2 digest */ + PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), + PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), +#endif + PY_HASH_ENTRY(NULL, NULL, NULL, 0), +}; + +static Py_uhash_t +py_hashentry_t_hash_name(const void *key) { + return _Py_HashBytes(key, strlen((const char *)key)); +} + +static int +py_hashentry_t_compare_name(const void *key1, const void *key2) { + return strcmp((const char *)key1, (const char *)key2); +} + +static void +py_hashentry_t_destroy_value(void *entry) { + py_hashentry_t *h = (py_hashentry_t *)entry; + if (--(h->refcnt) == 0) { + if (h->evp != NULL) { + PY_EVP_MD_free(h->evp); + h->evp = NULL; + } + if (h->evp_nosecurity != NULL) { + PY_EVP_MD_free(h->evp_nosecurity); + h->evp_nosecurity = NULL; + } + PyMem_Free(entry); + } +} + +static _Py_hashtable_t * +py_hashentry_table_new(void) { + _Py_hashtable_t *ht = _Py_hashtable_new_full( + py_hashentry_t_hash_name, + py_hashentry_t_compare_name, + NULL, + py_hashentry_t_destroy_value, + NULL + ); + if (ht == NULL) { + return NULL; + } + + for (const py_hashentry_t *h = py_hashes; h->py_name != NULL; h++) { + py_hashentry_t *entry = (py_hashentry_t *)PyMem_Malloc(sizeof(py_hashentry_t)); + if (entry == NULL) { + goto error; + } + memcpy(entry, h, sizeof(py_hashentry_t)); + + if (_Py_hashtable_set(ht, (const void*)entry->py_name, (void*)entry) < 0) { + PyMem_Free(entry); + goto error; + } + entry->refcnt = 1; + + if (h->py_alias != NULL) { + if (_Py_hashtable_set(ht, (const void*)entry->py_alias, (void*)entry) < 0) { + PyMem_Free(entry); + goto error; + } + entry->refcnt++; + } + } + + return ht; + error: + _Py_hashtable_destroy(ht); + return NULL; +} + +/* Module state */ static PyModuleDef _hashlibmodule; typedef struct { @@ -59,6 +230,7 @@ typedef struct { #endif PyObject *constructs; PyObject *unsupported_digestmod_error; + _Py_hashtable_t *hashtable; } _hashlibstate; static inline _hashlibstate* @@ -127,68 +299,15 @@ py_digest_name(const EVP_MD *md) { int nid = EVP_MD_nid(md); const char *name = NULL; + const py_hashentry_t *h; - /* Hard-coded names for well-known hashing algorithms. - * OpenSSL uses slightly different names algorithms like SHA3. - */ - switch (nid) { - case NID_md5: - name = "md5"; - break; - case NID_sha1: - name = "sha1"; - break; - case NID_sha224: - name ="sha224"; - break; - case NID_sha256: - name ="sha256"; - break; - case NID_sha384: - name ="sha384"; - break; - case NID_sha512: - name ="sha512"; - break; -#ifdef NID_sha512_224 - case NID_sha512_224: - name ="sha512_224"; - break; - case NID_sha512_256: - name ="sha512_256"; - break; -#endif -#ifdef PY_OPENSSL_HAS_SHA3 - case NID_sha3_224: - name ="sha3_224"; - break; - case NID_sha3_256: - name ="sha3_256"; - break; - case NID_sha3_384: - name ="sha3_384"; - break; - case NID_sha3_512: - name ="sha3_512"; - break; -#endif -#ifdef PY_OPENSSL_HAS_SHAKE - case NID_shake128: - name ="shake_128"; - break; - case NID_shake256: - name ="shake_256"; - break; -#endif -#ifdef PY_OPENSSL_HAS_BLAKE2 - case NID_blake2s256: - name ="blake2s"; - break; - case NID_blake2b512: - name ="blake2b"; - break; -#endif - default: + for (h = py_hashes; h->py_name != NULL; h++) { + if (h->ossl_nid == nid) { + name = h->py_name; + break; + } + } + if (name == NULL) { /* Ignore aliased names and only use long, lowercase name. The aliases * pollute the list and OpenSSL appears to have its own definition of * alias as the resulting list still contains duplicate and alternate @@ -197,67 +316,58 @@ py_digest_name(const EVP_MD *md) name = OBJ_nid2ln(nid); if (name == NULL) name = OBJ_nid2sn(nid); - break; } return PyUnicode_FromString(name); } -static const EVP_MD* -py_digest_by_name(const char *name) +/* Get EVP_MD by HID and purpose */ +static PY_EVP_MD* +py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) { - const EVP_MD *digest = EVP_get_digestbyname(name); + PY_EVP_MD *digest = NULL; + _hashlibstate *state = get_hashlib_state(module); + py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( + state->hashtable, (const void*)name + ); - /* OpenSSL uses dash instead of underscore in names of some algorithms - * like SHA3 and SHAKE. Detect different spellings. */ - if (digest == NULL) { - if (0) {} -#ifdef NID_sha512_224 - else if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) { - digest = EVP_sha512_224(); - } - else if (!strcmp(name, "sha512_256") || !strcmp(name, "SHA512_256")) { - digest = EVP_sha512_256(); - } -#endif -#ifdef PY_OPENSSL_HAS_SHA3 - /* could be sha3_ or shake_, Python never defined upper case */ - else if (!strcmp(name, "sha3_224")) { - digest = EVP_sha3_224(); - } - else if (!strcmp(name, "sha3_256")) { - digest = EVP_sha3_256(); - } - else if (!strcmp(name, "sha3_384")) { - digest = EVP_sha3_384(); - } - else if (!strcmp(name, "sha3_512")) { - digest = EVP_sha3_512(); - } -#endif -#ifdef PY_OPENSSL_HAS_SHAKE - else if (!strcmp(name, "shake_128")) { - digest = EVP_shake128(); - } - else if (!strcmp(name, "shake_256")) { - digest = EVP_shake256(); + if (entry != NULL) { + switch (py_ht) { + case Py_ht_evp: + case Py_ht_mac: + case Py_ht_pbkdf2: + if (entry->evp == NULL) { + entry->evp = PY_EVP_MD_fetch(entry->ossl_name, NULL); + } + digest = entry->evp; + break; + case Py_ht_evp_nosecurity: + if (entry->evp_nosecurity == NULL) { + entry->evp_nosecurity = PY_EVP_MD_fetch(entry->ossl_name, "-fips"); + } + digest = entry->evp_nosecurity; + break; } -#endif -#ifdef PY_OPENSSL_HAS_BLAKE2 - else if (!strcmp(name, "blake2s256")) { - digest = EVP_blake2s256(); + if (digest != NULL) { + PY_EVP_MD_up_ref(digest); } - else if (!strcmp(name, "blake2b512")) { - digest = EVP_blake2b512(); + } else { + // fall back for OpenSSL + switch (py_ht) { + case Py_ht_evp: + case Py_ht_mac: + case Py_ht_pbkdf2: + digest = PY_EVP_MD_fetch(name, NULL); + break; + case Py_ht_evp_nosecurity: + digest = PY_EVP_MD_fetch(name, "-fips"); + break; } -#endif } - if (digest == NULL) { PyErr_Format(PyExc_ValueError, "unsupported hash type %s", name); return NULL; } - return digest; } @@ -268,9 +378,9 @@ py_digest_by_name(const char *name) * * on error returns NULL with exception set. */ -static const EVP_MD* -py_digest_by_digestmod(PyObject *module, PyObject *digestmod) { - const EVP_MD* evp; +static PY_EVP_MD* +py_digest_by_digestmod(PyObject *module, PyObject *digestmod, enum Py_hash_type py_ht) { + PY_EVP_MD* evp; PyObject *name_obj = NULL; const char *name; @@ -295,7 +405,7 @@ py_digest_by_digestmod(PyObject *module, PyObject *digestmod) { return NULL; } - evp = py_digest_by_name(name); + evp = py_digest_by_name(module, name, py_ht); if (evp == NULL) { return NULL; } @@ -749,11 +859,11 @@ static PyType_Spec EVPXOFtype_spec = { #endif static PyObject * -EVPnew(PyObject *module, const EVP_MD *digest, +EVPnew(PyObject *module, PY_EVP_MD *digest, const unsigned char *cp, Py_ssize_t len, int usedforsecurity) { int result = 0; - EVPobject *self; + EVPobject *self = NULL; PyTypeObject *type = get_hashlib_state(module)->EVPtype; if (!digest) { @@ -767,8 +877,10 @@ EVPnew(PyObject *module, const EVP_MD *digest, } #endif - if ((self = newEVPobject(type)) == NULL) + if ((self = newEVPobject(type)) == NULL) { + PY_EVP_MD_free(digest); return NULL; + } if (!usedforsecurity) { #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW @@ -776,8 +888,9 @@ EVPnew(PyObject *module, const EVP_MD *digest, #endif } - - if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) { + result = EVP_DigestInit_ex(self->ctx, digest, NULL); + PY_EVP_MD_free(digest); + if (!result) { _setException(PyExc_ValueError); Py_DECREF(self); return NULL; @@ -800,6 +913,33 @@ EVPnew(PyObject *module, const EVP_MD *digest, return (PyObject *)self; } +static PyObject* +EVP_raw_new(PyObject *module, PyObject *data_obj, const char *name, + int usedforsecurity) +{ + Py_buffer view = { 0 }; + PyObject *ret_obj = NULL; + + if (data_obj) + GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); + + PY_EVP_MD *digest = py_digest_by_name( + module, name, + usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity + ); + + if (digest != NULL) { + ret_obj = EVPnew(module, digest, + (unsigned char*)view.buf, view.len, + usedforsecurity); + } + + if (data_obj) + PyBuffer_Release(&view); + + return ret_obj; +} + /* The module-level function: new() */ @@ -824,53 +964,14 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=ddd5053f92dffe90 input=c24554d0337be1b0]*/ { - Py_buffer view = { 0 }; - PyObject *ret_obj = NULL; char *name; - const EVP_MD *digest = NULL; - if (!PyArg_Parse(name_obj, "s", &name)) { PyErr_SetString(PyExc_TypeError, "name must be a string"); return NULL; } - - if (data_obj) - GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); - - digest = py_digest_by_name(name); - if (digest == NULL) { - goto exit; - } - - ret_obj = EVPnew(module, digest, - (unsigned char*)view.buf, view.len, - usedforsecurity); - -exit: - if (data_obj) - PyBuffer_Release(&view); - return ret_obj; + return EVP_raw_new(module, data_obj, name, usedforsecurity); } -static PyObject* -EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest, - int usedforsecurity) -{ - Py_buffer view = { 0 }; - PyObject *ret_obj; - - if (data_obj) - GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); - - ret_obj = EVPnew(module, digest, - (unsigned char*)view.buf, view.len, - usedforsecurity); - - if (data_obj) - PyBuffer_Release(&view); - - return ret_obj; -} /*[clinic input] _hashlib.openssl_md5 @@ -888,7 +989,7 @@ _hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/ { - return EVP_fast_new(module, data_obj, EVP_md5(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_md5, usedforsecurity); } @@ -908,7 +1009,7 @@ _hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=6813024cf690670d input=948f2f4b6deabc10]*/ { - return EVP_fast_new(module, data_obj, EVP_sha1(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha1, usedforsecurity); } @@ -928,7 +1029,7 @@ _hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=a2dfe7cc4eb14ebb input=f9272821fadca505]*/ { - return EVP_fast_new(module, data_obj, EVP_sha224(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha224, usedforsecurity); } @@ -948,7 +1049,7 @@ _hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=1f874a34870f0a68 input=549fad9d2930d4c5]*/ { - return EVP_fast_new(module, data_obj, EVP_sha256(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha256, usedforsecurity); } @@ -968,7 +1069,7 @@ _hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=58529eff9ca457b2 input=48601a6e3bf14ad7]*/ { - return EVP_fast_new(module, data_obj, EVP_sha384(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha384, usedforsecurity); } @@ -988,7 +1089,7 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=2c744c9e4a40d5f6 input=c5c46a2a817aa98f]*/ { - return EVP_fast_new(module, data_obj, EVP_sha512(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha512, usedforsecurity); } @@ -1010,7 +1111,7 @@ _hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=144641c1d144b974 input=e3a01b2888916157]*/ { - return EVP_fast_new(module, data_obj, EVP_sha3_224(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha3_224, usedforsecurity); } /*[clinic input] @@ -1029,7 +1130,7 @@ _hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=c61f1ab772d06668 input=e2908126c1b6deed]*/ { - return EVP_fast_new(module, data_obj, EVP_sha3_256(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha3_256, usedforsecurity); } /*[clinic input] @@ -1048,7 +1149,7 @@ _hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=f68e4846858cf0ee input=ec0edf5c792f8252]*/ { - return EVP_fast_new(module, data_obj, EVP_sha3_384(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha3_384, usedforsecurity); } /*[clinic input] @@ -1067,7 +1168,7 @@ _hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=2eede478c159354a input=64e2cc0c094d56f4]*/ { - return EVP_fast_new(module, data_obj, EVP_sha3_512(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_sha3_512, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHA3 */ @@ -1088,7 +1189,7 @@ _hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=bc49cdd8ada1fa97 input=6c9d67440eb33ec8]*/ { - return EVP_fast_new(module, data_obj, EVP_shake128(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_shake_128, usedforsecurity); } /*[clinic input] @@ -1107,7 +1208,7 @@ _hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=358d213be8852df7 input=479cbe9fefd4a9f8]*/ { - return EVP_fast_new(module, data_obj, EVP_shake256(), usedforsecurity); + return EVP_raw_new(module, data_obj, Py_hash_shake_256, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHAKE */ @@ -1133,9 +1234,9 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, char *key; long dklen; int retval; - const EVP_MD *digest; + PY_EVP_MD *digest = NULL; - digest = py_digest_by_name(hash_name); + digest = py_digest_by_name(module, hash_name, Py_ht_pbkdf2); if (digest == NULL) { goto end; } @@ -1203,6 +1304,9 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, } end: + if (digest != NULL) { + PY_EVP_MD_free(digest); + } return key_obj; } @@ -1352,12 +1456,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, unsigned char md[EVP_MAX_MD_SIZE] = {0}; unsigned int md_len = 0; unsigned char *result; - const EVP_MD *evp; - - evp = py_digest_by_digestmod(module, digest); - if (evp == NULL) { - return NULL; - } + PY_EVP_MD *evp; if (key->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, @@ -1370,6 +1469,11 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, return NULL; } + evp = py_digest_by_digestmod(module, digest, Py_ht_mac); + if (evp == NULL) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS result = HMAC( evp, @@ -1378,6 +1482,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, md, &md_len ); Py_END_ALLOW_THREADS + PY_EVP_MD_free(evp); if (result == NULL) { _setException(PyExc_ValueError); @@ -1407,7 +1512,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, /*[clinic end generated code: output=c20d9e4d9ed6d219 input=5f4071dcc7f34362]*/ { PyTypeObject *type = get_hashlib_state(module)->HMACtype; - const EVP_MD *digest; + PY_EVP_MD *digest; HMAC_CTX *ctx = NULL; HMACobject *self = NULL; int r; @@ -1424,7 +1529,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, return NULL; } - digest = py_digest_by_digestmod(module, digestmod); + digest = py_digest_by_digestmod(module, digestmod, Py_ht_mac); if (digest == NULL) { return NULL; } @@ -1441,6 +1546,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, (int)key->len, digest, NULL /*impl*/); + PY_EVP_MD_free(digest); if (r == 0) { _setException(PyExc_ValueError); goto error; @@ -2004,6 +2110,12 @@ hashlib_clear(PyObject *m) #endif Py_CLEAR(state->constructs); Py_CLEAR(state->unsupported_digestmod_error); + + if (state->hashtable != NULL) { + _Py_hashtable_destroy(state->hashtable); + state->hashtable = NULL; + } + return 0; } @@ -2014,6 +2126,19 @@ hashlib_free(void *m) } /* Py_mod_exec functions */ +static int +hashlib_init_hashtable(PyObject *module) +{ + _hashlibstate *state = get_hashlib_state(module); + + state->hashtable = py_hashentry_table_new(); + if (state->hashtable == NULL) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + static int hashlib_init_evptype(PyObject *module) { @@ -2141,6 +2266,7 @@ hashlib_exception(PyObject *module) static PyModuleDef_Slot hashlib_slots[] = { + {Py_mod_exec, hashlib_init_hashtable}, {Py_mod_exec, hashlib_init_evptype}, {Py_mod_exec, hashlib_init_evpxoftype}, {Py_mod_exec, hashlib_init_hmactype}, From d02100944834f3c09adf935dce8ba740746714d1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 15:33:55 +0100 Subject: [PATCH 02/10] Potentially better error message for scrypt --- Modules/_hashopenssl.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 30bb5bdd6b43e4..d0c2a4c0e4098d 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -1405,9 +1405,14 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, /* let OpenSSL validate the rest */ retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, maxmem, NULL, 0); if (!retval) { - /* sorry, can't do much better */ - PyErr_SetString(PyExc_ValueError, - "Invalid parameter combination for n, r, p, maxmem."); + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + _setException(PyExc_ValueError); + } else { + /* sorry, can't do much better */ + PyErr_SetString(PyExc_ValueError, + "Invalid parameter combination for n, r, p, maxmem."); + } return NULL; } From b58fde5755a6f1e3c8dd011f019adaae1e9d8097 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 15:41:22 +0100 Subject: [PATCH 03/10] scrypt is blocked in FIPS mode --- Lib/test/test_hashlib.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 1623bf350e2875..28d017a47987fd 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -48,12 +48,15 @@ builtin_hashlib = None try: - from _hashlib import HASH, HASHXOF, openssl_md_meth_names + from _hashlib import HASH, HASHXOF, openssl_md_meth_names, get_fips_mode except ImportError: HASH = None HASHXOF = None openssl_md_meth_names = frozenset() + def get_fips_mode(): + return 0 + try: import _blake2 except ImportError: @@ -192,10 +195,7 @@ def hash_constructors(self): @property def is_fips_mode(self): - if hasattr(self._hashlib, "get_fips_mode"): - return self._hashlib.get_fips_mode() - else: - return None + return get_fips_mode() def test_hash_array(self): a = array.array("b", range(10)) @@ -1057,6 +1057,7 @@ def test_pbkdf2_hmac_c(self): @unittest.skipUnless(hasattr(hashlib, 'scrypt'), ' test requires OpenSSL > 1.1') + @unittest.skipIf(get_fips_mode(), reason="scrypt is blocked in FIPS mode") def test_scrypt(self): for password, salt, n, r, p, expected in self.scrypt_test_vectors: result = hashlib.scrypt(password, salt=salt, n=n, r=r, p=p) From 1668f6692e7da0d90e7c71b8e962cdbb846457bf Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 15:58:33 +0100 Subject: [PATCH 04/10] Better error reporting --- Modules/_hashopenssl.c | 86 +++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index d0c2a4c0e4098d..f4b19bb35c6276 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -265,16 +265,26 @@ class _hashlib.HMAC "HMACobject *" "((_hashlibstate *)PyModule_GetState(module)) /* LCOV_EXCL_START */ static PyObject * -_setException(PyObject *exc) +_setException(PyObject *exc, const char* altmsg, ...) { - unsigned long errcode; + unsigned long errcode = ERR_peek_last_error(); const char *lib, *func, *reason; + va_list vargs; - errcode = ERR_peek_last_error(); +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, altmsg); +#else + va_start(vargs); +#endif if (!errcode) { - PyErr_SetString(exc, "unknown reasons"); + if (altmsg == NULL) { + PyErr_SetString(exc, "unknown reasons"); + } else { + PyErr_FormatV(exc, altmsg, vargs); + } return NULL; } + va_end(vargs); ERR_clear_error(); lib = ERR_lib_error_string(errcode); @@ -365,7 +375,7 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) } } if (digest == NULL) { - PyErr_Format(PyExc_ValueError, "unsupported hash type %s", name); + _setException(PyExc_ValueError, "unsupported hash type %s", name); return NULL; } return digest; @@ -444,7 +454,7 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len) else process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int); if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return -1; } len -= process; @@ -495,7 +505,7 @@ EVP_copy_impl(EVPobject *self) if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) { Py_DECREF(newobj); - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } return (PyObject *)newobj; } @@ -522,11 +532,11 @@ EVP_digest_impl(EVPobject *self) } if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } digest_size = EVP_MD_CTX_size(temp_ctx); if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return NULL; } @@ -557,11 +567,11 @@ EVP_hexdigest_impl(EVPobject *self) /* Get the raw (binary) digest value */ if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } digest_size = EVP_MD_CTX_size(temp_ctx); if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return NULL; } @@ -737,14 +747,14 @@ EVPXOF_digest_impl(EVPobject *self, Py_ssize_t length) if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { Py_DECREF(retval); EVP_MD_CTX_free(temp_ctx); - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } if (!EVP_DigestFinalXOF(temp_ctx, (unsigned char*)PyBytes_AS_STRING(retval), length)) { Py_DECREF(retval); EVP_MD_CTX_free(temp_ctx); - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return NULL; } @@ -785,12 +795,12 @@ EVPXOF_hexdigest_impl(EVPobject *self, Py_ssize_t length) if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { PyMem_Free(digest); EVP_MD_CTX_free(temp_ctx); - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } if (!EVP_DigestFinalXOF(temp_ctx, digest, length)) { PyMem_Free(digest); EVP_MD_CTX_free(temp_ctx); - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return NULL; } @@ -866,10 +876,7 @@ EVPnew(PyObject *module, PY_EVP_MD *digest, EVPobject *self = NULL; PyTypeObject *type = get_hashlib_state(module)->EVPtype; - if (!digest) { - PyErr_SetString(PyExc_ValueError, "unsupported hash type"); - return NULL; - } + assert(digest != NULL); #ifdef PY_OPENSSL_HAS_SHAKE if ((EVP_MD_flags(digest) & EVP_MD_FLAG_XOF) == EVP_MD_FLAG_XOF) { @@ -891,7 +898,7 @@ EVPnew(PyObject *module, PY_EVP_MD *digest, result = EVP_DigestInit_ex(self->ctx, digest, NULL); PY_EVP_MD_free(digest); if (!result) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); Py_DECREF(self); return NULL; } @@ -1299,7 +1306,7 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, if (!retval) { Py_CLEAR(key_obj); - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); goto end; } @@ -1405,14 +1412,7 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, /* let OpenSSL validate the rest */ retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, maxmem, NULL, 0); if (!retval) { - unsigned long errcode = ERR_peek_last_error(); - if (errcode) { - _setException(PyExc_ValueError); - } else { - /* sorry, can't do much better */ - PyErr_SetString(PyExc_ValueError, - "Invalid parameter combination for n, r, p, maxmem."); - } + _setException(PyExc_ValueError, "Invalid parameter combination for n, r, p, maxmem."); return NULL; } @@ -1433,7 +1433,7 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, if (!retval) { Py_CLEAR(key_obj); - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return NULL; } return key_obj; @@ -1490,7 +1490,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, PY_EVP_MD_free(evp); if (result == NULL) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return NULL; } return PyBytes_FromStringAndSize((const char*)md, md_len); @@ -1541,7 +1541,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, ctx = HMAC_CTX_new(); if (ctx == NULL) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); goto error; } @@ -1553,7 +1553,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, NULL /*impl*/); PY_EVP_MD_free(digest); if (r == 0) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); goto error; } @@ -1623,7 +1623,7 @@ _hmac_update(HMACobject *self, PyObject *obj) PyBuffer_Release(&view); if (r == 0) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return 0; } return 1; @@ -1643,11 +1643,11 @@ _hashlib_HMAC_copy_impl(HMACobject *self) HMAC_CTX *ctx = HMAC_CTX_new(); if (ctx == NULL) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } if (!locked_HMAC_CTX_copy(ctx, self)) { HMAC_CTX_free(ctx); - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } retval = (HMACobject *)PyObject_New(HMACobject, Py_TYPE(self)); @@ -1713,13 +1713,13 @@ _hmac_digest(HMACobject *self, unsigned char *buf, unsigned int len) return 0; } if (!locked_HMAC_CTX_copy(temp_ctx, self)) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return 0; } int r = HMAC_Final(temp_ctx, buf, &len); HMAC_CTX_free(temp_ctx); if (r == 0) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return 0; } return 1; @@ -1737,7 +1737,7 @@ _hashlib_HMAC_digest_impl(HMACobject *self) unsigned char digest[EVP_MAX_MD_SIZE]; unsigned int digest_size = _hmac_digest_size(self); if (digest_size == 0) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } int r = _hmac_digest(self, digest, digest_size); if (r == 0) { @@ -1762,7 +1762,7 @@ _hashlib_HMAC_hexdigest_impl(HMACobject *self) unsigned char digest[EVP_MAX_MD_SIZE]; unsigned int digest_size = _hmac_digest_size(self); if (digest_size == 0) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } int r = _hmac_digest(self, digest, digest_size); if (r == 0) { @@ -1776,7 +1776,7 @@ _hashlib_hmac_get_digest_size(HMACobject *self, void *closure) { unsigned int digest_size = _hmac_digest_size(self); if (digest_size == 0) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } return PyLong_FromLong(digest_size); } @@ -1786,7 +1786,7 @@ _hashlib_hmac_get_block_size(HMACobject *self, void *closure) { const EVP_MD *md = HMAC_CTX_get_md(self->ctx); if (md == NULL) { - return _setException(PyExc_ValueError); + return _setException(PyExc_ValueError, NULL); } return PyLong_FromLong(EVP_MD_block_size(md)); } @@ -1939,7 +1939,7 @@ _hashlib_get_fips_mode_impl(PyObject *module) // But 0 is also a valid result value. unsigned long errcode = ERR_peek_last_error(); if (errcode) { - _setException(PyExc_ValueError); + _setException(PyExc_ValueError, NULL); return -1; } } From cf68e659a250be40582aaa9f8249be24321b081c Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 16:14:41 +0100 Subject: [PATCH 05/10] address review comment --- Modules/_hashopenssl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index f4b19bb35c6276..f2f0d2db77149c 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -1241,9 +1241,8 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, char *key; long dklen; int retval; - PY_EVP_MD *digest = NULL; - digest = py_digest_by_name(module, hash_name, Py_ht_pbkdf2); + PY_EVP_MD *digest = py_digest_by_name(module, hash_name, Py_ht_pbkdf2); if (digest == NULL) { goto end; } From 487becdac0fe866d319b3a545ab653ff4a101b08 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 16:14:58 +0100 Subject: [PATCH 06/10] Fix tests, need to check for openssl digest --- Lib/test/test_hashlib.py | 2 +- Lib/test/test_imaplib.py | 6 +++--- Lib/test/test_poplib.py | 4 ++-- Lib/test/test_smtplib.py | 4 ++-- Lib/test/test_tools/test_md5sum.py | 2 +- Lib/test/test_urllib2_localnet.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 28d017a47987fd..110eb48fd4f8ca 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -1017,7 +1017,7 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): self.assertEqual(out, expected, (digest_name, password, salt, rounds)) - with self.assertRaisesRegex(ValueError, 'unsupported hash type'): + with self.assertRaisesRegex(ValueError, '.*unsupported.*'): pbkdf2('unknown', b'pass', b'salt', 1) if 'sha1' in supported: diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index c2b935f58164e5..30b553746af11a 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -387,7 +387,7 @@ def cmd_AUTHENTICATE(self, tag, args): self.assertEqual(code, 'OK') self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5_bytes(self): class AuthHandler(SimpleIMAPHandler): capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' @@ -405,7 +405,7 @@ def cmd_AUTHENTICATE(self, tag, args): ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf") self.assertEqual(ret, "OK") - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5_plain_text(self): class AuthHandler(SimpleIMAPHandler): capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' @@ -851,7 +851,7 @@ def cmd_AUTHENTICATE(self, tag, args): b'ZmFrZQ==\r\n') # b64 encoded 'fake' @threading_helper.reap_threads - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5(self): class AuthHandler(SimpleIMAPHandler): diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 44cf5231f9d23f..1220ca32ef82e8 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -318,11 +318,11 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_apop_REDOS(self): # Replace welcome with very long evil welcome. # NB The upper bound on welcome length is currently 2048. diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 9761a372517315..1a60fef8a428b7 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -1171,7 +1171,7 @@ def auth_buggy(challenge=None): finally: smtp.close() - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_CRAM_MD5(self): self.serv.add_feature("AUTH CRAM-MD5") smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', @@ -1180,7 +1180,7 @@ def testAUTH_CRAM_MD5(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() - @hashlib_helper.requires_hashdigest('md5') + @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_multiple(self): # Test that multiple authentication methods are tried. self.serv.add_feature("AUTH BOGUS PLAIN LOGIN CRAM-MD5") diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py index 92315f181c82cd..c5a230e95c2b7e 100644 --- a/Lib/test/test_tools/test_md5sum.py +++ b/Lib/test/test_tools/test_md5sum.py @@ -11,7 +11,7 @@ skip_if_missing() -@hashlib_helper.requires_hashdigest('md5') +@hashlib_helper.requires_hashdigest('md5', openssl=True) class MD5SumTests(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 36fb05d3db0e2a..0b2d07ce61d5c7 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -317,7 +317,7 @@ def test_basic_auth_httperror(self): self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) -@hashlib_helper.requires_hashdigest("md5") +@hashlib_helper.requires_hashdigest("md5", openssl=True) class ProxyAuthTests(unittest.TestCase): URL = "http://localhost" From 12066998ba65432c38aac8dd40fa6c928c82260c Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 16:47:48 +0100 Subject: [PATCH 07/10] sha512_256 is not always available --- Doc/library/hashlib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 0c3bd7b5ac2c99..53320d9cb0d6c3 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -120,10 +120,10 @@ More condensed: Using :func:`new` with an algorithm provided by OpenSSL: - >>> h = hashlib.new('sha512_256') + >>> h = hashlib.new('sha256') >>> h.update(b"Nobody inspects the spammish repetition") >>> h.hexdigest() - '19197dc4d03829df858011c6c87600f994a858103bbc19005f20987aa19a97e2' + '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' Hashlib provides the following constant attributes: From ae88881cd19810b44ef5971f18edd810afbdf737 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 8 Jan 2022 13:26:19 +0100 Subject: [PATCH 08/10] Enable cache, compare name has to return 1 on success --- Modules/_hashopenssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index f2f0d2db77149c..05ddf1875364da 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -159,7 +159,7 @@ py_hashentry_t_hash_name(const void *key) { static int py_hashentry_t_compare_name(const void *key1, const void *key2) { - return strcmp((const char *)key1, (const char *)key2); + return strcmp((const char *)key1, (const char *)key2) == 0; } static void From 733b7896c88d379184562c38bd7305719e1063fc Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 8 Jan 2022 13:36:53 +0100 Subject: [PATCH 09/10] Address review comment --- Modules/_hashopenssl.c | 142 ++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 81 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 05ddf1875364da..6609565bd45669 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -84,7 +84,7 @@ typedef struct { const char *py_alias; const char *ossl_name; int ossl_nid; - Py_ssize_t refcnt; + int refcnt; PY_EVP_MD *evp; PY_EVP_MD *evp_nosecurity; } py_hashentry_t; @@ -95,27 +95,19 @@ typedef struct { #define Py_hash_sha256 "sha256" #define Py_hash_sha384 "sha384" #define Py_hash_sha512 "sha512" -#ifdef NID_sha512_224 #define Py_hash_sha512_224 "sha512_224" #define Py_hash_sha512_256 "sha512_256" -#endif -#ifdef PY_OPENSSL_HAS_SHA3 #define Py_hash_sha3_224 "sha3_224" #define Py_hash_sha3_256 "sha3_256" #define Py_hash_sha3_384 "sha3_384" #define Py_hash_sha3_512 "sha3_512" -#endif -#ifdef PY_OPENSSL_HAS_SHAKE #define Py_hash_shake_128 "shake_128" #define Py_hash_shake_256 "shake_256" -#endif -#ifdef PY_OPENSSL_HAS_BLAKE2 #define Py_hash_blake2s "blake2s" #define Py_hash_blake2b "blake2b" -#endif #define PY_HASH_ENTRY(py_name, py_alias, ossl_name, ossl_nid) \ - {py_name, py_alias, ossl_name, ossl_nid, 0, NULL} + {py_name, py_alias, ossl_name, ossl_nid, 0, NULL, NULL} static const py_hashentry_t py_hashes[] = { /* md5 */ @@ -127,28 +119,20 @@ static const py_hashentry_t py_hashes[] = { PY_HASH_ENTRY(Py_hash_sha256, "SHA256", SN_sha256, NID_sha256), PY_HASH_ENTRY(Py_hash_sha384, "SHA384", SN_sha384, NID_sha384), PY_HASH_ENTRY(Py_hash_sha512, "SHA512", SN_sha512, NID_sha512), -#ifdef NID_sha512_224 /* truncated sha2 */ PY_HASH_ENTRY(Py_hash_sha512_224, "SHA512_224", SN_sha512_224, NID_sha512_224), PY_HASH_ENTRY(Py_hash_sha512_256, "SHA512_256", SN_sha512_256, NID_sha512_256), -#endif -#ifdef PY_OPENSSL_HAS_SHA3 /* sha3 */ PY_HASH_ENTRY(Py_hash_sha3_224, NULL, SN_sha3_224, NID_sha3_224), PY_HASH_ENTRY(Py_hash_sha3_256, NULL, SN_sha3_256, NID_sha3_256), PY_HASH_ENTRY(Py_hash_sha3_384, NULL, SN_sha3_384, NID_sha3_384), PY_HASH_ENTRY(Py_hash_sha3_512, NULL, SN_sha3_512, NID_sha3_512), -#endif -#ifdef PY_OPENSSL_HAS_SHAKE /* sha3 shake */ PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), -#endif -#ifdef PY_OPENSSL_HAS_BLAKE2 /* blake2 digest */ PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), -#endif PY_HASH_ENTRY(NULL, NULL, NULL, 0), }; @@ -278,7 +262,7 @@ _setException(PyObject *exc, const char* altmsg, ...) #endif if (!errcode) { if (altmsg == NULL) { - PyErr_SetString(exc, "unknown reasons"); + PyErr_SetString(exc, "no reason supplied"); } else { PyErr_FormatV(exc, altmsg, vargs); } @@ -336,10 +320,14 @@ static PY_EVP_MD* py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) { PY_EVP_MD *digest = NULL; +#if 1 _hashlibstate *state = get_hashlib_state(module); py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( state->hashtable, (const void*)name ); +#else + py_hashentry_t *entry = NULL; +#endif if (entry != NULL) { switch (py_ht) { @@ -362,7 +350,7 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) PY_EVP_MD_up_ref(digest); } } else { - // fall back for OpenSSL + // Fall back for looking up an unindexed OpenSSL specific name. switch (py_ht) { case Py_ht_evp: case Py_ht_mac: @@ -868,83 +856,75 @@ static PyType_Spec EVPXOFtype_spec = { #endif -static PyObject * -EVPnew(PyObject *module, PY_EVP_MD *digest, - const unsigned char *cp, Py_ssize_t len, int usedforsecurity) +static PyObject* +py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, + int usedforsecurity) { - int result = 0; + Py_buffer view = { 0 }; + PY_EVP_MD *digest = NULL; + PyTypeObject *type; EVPobject *self = NULL; - PyTypeObject *type = get_hashlib_state(module)->EVPtype; - assert(digest != NULL); + if (data_obj != NULL) { + GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); + } + + digest = py_digest_by_name( + module, digestname, usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity + ); + if (digest == NULL) { + goto exit; + } -#ifdef PY_OPENSSL_HAS_SHAKE if ((EVP_MD_flags(digest) & EVP_MD_FLAG_XOF) == EVP_MD_FLAG_XOF) { type = get_hashlib_state(module)->EVPXOFtype; + } else { + type = get_hashlib_state(module)->EVPtype; } -#endif - if ((self = newEVPobject(type)) == NULL) { - PY_EVP_MD_free(digest); - return NULL; + self = newEVPobject(type); + if (self == NULL) { + goto exit; } +#if defined(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW) && OPENSSL_VERSION_NUMBER >= 0x30000000L + // In OpenSSL 1.1.1 the non FIPS allowed flag is context specific while + // in 3.0.0 it is a different EVP_MD provider. if (!usedforsecurity) { -#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW EVP_MD_CTX_set_flags(self->ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); -#endif } +#endif - result = EVP_DigestInit_ex(self->ctx, digest, NULL); - PY_EVP_MD_free(digest); + int result = EVP_DigestInit_ex(self->ctx, digest, NULL); if (!result) { _setException(PyExc_ValueError, NULL); - Py_DECREF(self); - return NULL; + Py_CLEAR(self); + goto exit; } - if (cp && len) { - if (len >= HASHLIB_GIL_MINSIZE) { + if (view.buf && view.len) { + if (view.len >= HASHLIB_GIL_MINSIZE) { Py_BEGIN_ALLOW_THREADS - result = EVP_hash(self, cp, len); + result = EVP_hash(self, view.buf, view.len); Py_END_ALLOW_THREADS } else { - result = EVP_hash(self, cp, len); + result = EVP_hash(self, view.buf, view.len); } if (result == -1) { - Py_DECREF(self); - return NULL; + Py_CLEAR(self); + goto exit; } } - return (PyObject *)self; -} - -static PyObject* -EVP_raw_new(PyObject *module, PyObject *data_obj, const char *name, - int usedforsecurity) -{ - Py_buffer view = { 0 }; - PyObject *ret_obj = NULL; - - if (data_obj) - GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); - - PY_EVP_MD *digest = py_digest_by_name( - module, name, - usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity - ); - + exit: + if (data_obj != NULL) { + PyBuffer_Release(&view); + } if (digest != NULL) { - ret_obj = EVPnew(module, digest, - (unsigned char*)view.buf, view.len, - usedforsecurity); + PY_EVP_MD_free(digest); } - if (data_obj) - PyBuffer_Release(&view); - - return ret_obj; + return (PyObject *)self; } @@ -976,7 +956,7 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, PyErr_SetString(PyExc_TypeError, "name must be a string"); return NULL; } - return EVP_raw_new(module, data_obj, name, usedforsecurity); + return py_evp_fromname(module, name, data_obj, usedforsecurity); } @@ -996,7 +976,7 @@ _hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/ { - return EVP_raw_new(module, data_obj, Py_hash_md5, usedforsecurity); + return py_evp_fromname(module, Py_hash_md5, data_obj, usedforsecurity); } @@ -1016,7 +996,7 @@ _hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=6813024cf690670d input=948f2f4b6deabc10]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha1, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha1, data_obj, usedforsecurity); } @@ -1036,7 +1016,7 @@ _hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=a2dfe7cc4eb14ebb input=f9272821fadca505]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha224, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha224, data_obj, usedforsecurity); } @@ -1056,7 +1036,7 @@ _hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=1f874a34870f0a68 input=549fad9d2930d4c5]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha256, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha256, data_obj, usedforsecurity); } @@ -1076,7 +1056,7 @@ _hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=58529eff9ca457b2 input=48601a6e3bf14ad7]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha384, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha384, data_obj, usedforsecurity); } @@ -1096,7 +1076,7 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=2c744c9e4a40d5f6 input=c5c46a2a817aa98f]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha512, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha512, data_obj, usedforsecurity); } @@ -1118,7 +1098,7 @@ _hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=144641c1d144b974 input=e3a01b2888916157]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha3_224, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha3_224, data_obj, usedforsecurity); } /*[clinic input] @@ -1137,7 +1117,7 @@ _hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=c61f1ab772d06668 input=e2908126c1b6deed]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha3_256, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha3_256, data_obj , usedforsecurity); } /*[clinic input] @@ -1156,7 +1136,7 @@ _hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=f68e4846858cf0ee input=ec0edf5c792f8252]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha3_384, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha3_384, data_obj , usedforsecurity); } /*[clinic input] @@ -1175,7 +1155,7 @@ _hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=2eede478c159354a input=64e2cc0c094d56f4]*/ { - return EVP_raw_new(module, data_obj, Py_hash_sha3_512, usedforsecurity); + return py_evp_fromname(module, Py_hash_sha3_512, data_obj , usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHA3 */ @@ -1196,7 +1176,7 @@ _hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=bc49cdd8ada1fa97 input=6c9d67440eb33ec8]*/ { - return EVP_raw_new(module, data_obj, Py_hash_shake_128, usedforsecurity); + return py_evp_fromname(module, Py_hash_shake_128, data_obj , usedforsecurity); } /*[clinic input] @@ -1215,7 +1195,7 @@ _hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, int usedforsecurity) /*[clinic end generated code: output=358d213be8852df7 input=479cbe9fefd4a9f8]*/ { - return EVP_raw_new(module, data_obj, Py_hash_shake_256, usedforsecurity); + return py_evp_fromname(module, Py_hash_shake_256, data_obj , usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHAKE */ From a9e696fd156b984a0baf535427e25da0e14a6cb6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 9 Jan 2022 21:40:11 +0100 Subject: [PATCH 10/10] Remove perf check block --- Modules/_hashopenssl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 6609565bd45669..eeea61aeceb542 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -320,14 +320,10 @@ static PY_EVP_MD* py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) { PY_EVP_MD *digest = NULL; -#if 1 _hashlibstate *state = get_hashlib_state(module); py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( state->hashtable, (const void*)name ); -#else - py_hashentry_t *entry = NULL; -#endif if (entry != NULL) { switch (py_ht) {