Thanks to visit codestin.com
Credit goes to github.com

Skip to content

gh-135532: module-wide cleanup for HACL*-based extension modules #135536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 50 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
5d8c093
add common object head for hashlib/hmac objects
picnixz Jun 8, 2025
81e3046
simplify digest computation
picnixz Jun 8, 2025
7f9f7b7
refactor update logic
picnixz Jun 8, 2025
15a4f2f
refactor alloc() logic
picnixz Jun 8, 2025
5cd828a
finalizing touches
picnixz Jun 8, 2025
63db1de
correct mutex usage
picnixz Jun 15, 2025
ea033a3
Revert 5cd828acdcfef753aee5eec7e13f07682af40f46
picnixz Jun 15, 2025
77baa67
revert some constructor changes
picnixz Jun 15, 2025
7b7b1a3
use a macro for the mutex API
picnixz Jun 15, 2025
0dcc799
use 'state' attribute instead of 'hash_state'
picnixz Jun 15, 2025
5b71796
use '<MOD>state' and 'get_<MOD>module_state' naming
picnixz Jun 15, 2025
85ae8c3
use consistent '<mod>module_{exec,traverse,clear,free}' naming
picnixz Jun 15, 2025
64e1513
use consistent '<mod>module_{methods,slots,def}' naming
picnixz Jun 15, 2025
abfa1af
update clinic directives
picnixz Jun 15, 2025
902759f
unconditionally lock when performing HASH updates
picnixz Jun 16, 2025
dde68c4
Merge remote-tracking branch 'upstream/main' into perf/hashlib/mutex-…
picnixz Jun 16, 2025
05c1e66
post-merge
picnixz Jun 16, 2025
db57278
do not guard against empty buffers for now
picnixz Jun 16, 2025
ead20a1
consistency fixes
picnixz Jun 16, 2025
68a6bbc
remove unused import
picnixz Jun 16, 2025
68f297e
correct naming for locked/unlocked versions
picnixz Jun 16, 2025
9817c3d
debug?
picnixz Jun 16, 2025
7c6842b
Merge remote-tracking branch 'upstream/main' into perf/hashlib/mutex-…
picnixz Jun 16, 2025
c14c87d
simplify HMAC
picnixz Jun 16, 2025
bfb5436
release the GIL for large buffers
picnixz Jun 16, 2025
923c05f
restore GIL_MINSIZE
picnixz Jun 16, 2025
55b2afa
correctly lock objects
picnixz Jun 16, 2025
5cd60d1
improve tests
picnixz Jun 16, 2025
a2fcbd5
fixup HMAC
picnixz Jun 16, 2025
417cee1
fixup
picnixz Jun 16, 2025
f350501
GIL protection
picnixz Jun 16, 2025
5c4009d
show WASI errors
picnixz Jun 16, 2025
8aec797
fix WASI
picnixz Jun 16, 2025
6db58dc
fix compilation
picnixz Jun 16, 2025
b1f9463
fix compilation
picnixz Jun 16, 2025
491b922
fix warnings
picnixz Jun 16, 2025
c048975
sync
picnixz Jun 17, 2025
c9044d2
fixup format string
picnixz Jun 17, 2025
953d5bf
Merge branch 'main' into refactor/crypto/hacl-based-135532
picnixz Jun 17, 2025
3849f2d
pformat clinic directives
picnixz Jun 17, 2025
6c08f0d
address review
picnixz Jun 17, 2025
7fd1396
reudce diff
picnixz Jun 20, 2025
f400a11
Merge remote-tracking branch 'upstream/main' into perf/hashlib/mutex-…
picnixz Jun 20, 2025
5e2daa8
Merge remote-tracking branch 'upstream/main' into perf/hashlib/mutex-…
picnixz Jun 20, 2025
4f9729e
Merge remote-tracking branch 'upstream/main' into perf/hashlib/mutex-…
picnixz Jun 20, 2025
2ccccf0
Merge remote-tracking branch 'upstream/main' into refactor/crypto/hac…
picnixz Jun 20, 2025
42448db
Merge branch 'perf/hashlib/mutex-135239' into refactor/crypto/hacl-ba…
picnixz Jun 20, 2025
82cef2c
Merge branch 'main' into refactor/crypto/hacl-based-135532
picnixz Jun 22, 2025
d0cfb7c
post-merge
picnixz Jun 22, 2025
c86687f
diff reduce
picnixz Jun 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 92 additions & 91 deletions Modules/blake2module.c
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* Written in 2013 by Dmitry Chestnykh <[email protected]>
* Modified for CPython by Christian Heimes <[email protected]>
* Updated to use HACL* by Jonathan Protzenko <[email protected]>
* Refactored by Bénédikt Tran <[email protected]>
*
* To the extent possible under law, the author have dedicated all
* copyright and related and neighboring rights to this software to
Expand Down Expand Up @@ -64,69 +65,106 @@
#include "_hacl/Hacl_Hash_Blake2b_Simd256.h"
#endif

// MODULE TYPE SLOTS

static PyType_Spec blake2b_type_spec;
static PyType_Spec blake2s_type_spec;

PyDoc_STRVAR(blake2mod__doc__,
"_blake2 provides BLAKE2b and BLAKE2s for hashlib\n");
// --- BLAKE-2 module state ---------------------------------------------------

typedef struct {
PyTypeObject *blake2b_type;
PyTypeObject *blake2s_type;
bool can_run_simd128;
bool can_run_simd256;
} Blake2State;
} blake2module_state;

static inline Blake2State *
blake2_get_state(PyObject *module)
static inline blake2module_state *
get_blake2module_state(PyObject *module)
{
void *state = _PyModule_GetState(module);
assert(state != NULL);
return (Blake2State *)state;
return (blake2module_state *)state;
}

#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256)
static inline Blake2State *
blake2_get_state_from_type(PyTypeObject *module)
static inline blake2module_state *
get_blake2module_state_by_cls(PyTypeObject *cls)
{
void *state = _PyType_GetModuleState(module);
void *state = _PyType_GetModuleState(cls);
assert(state != NULL);
return (Blake2State *)state;
return (blake2module_state *)state;
}

// --- BLAKE-2 object ---------------------------------------------------------

// The HACL* API does not offer an agile API that can deal with either Blake2S
// or Blake2B -- the reason is that the underlying states are optimized (uint32s
// for S, uint64s for B). Therefore, we use a tagged union in this module to
// correctly dispatch. Note that the previous incarnation of this code
// transformed the Blake2b implementation into the Blake2s one using a script,
// so this is an improvement.
//
// The 128 and 256 versions are only available if i) we were able to compile
// them, and ii) if the CPU we run on also happens to have the right instruction
// set.
typedef enum { Blake2s, Blake2b, Blake2s_128, Blake2b_256 } blake2_impl;

typedef struct {
HASHLIB_OBJECT_HEAD
union {
Hacl_Hash_Blake2s_state_t *blake2s_state;
Hacl_Hash_Blake2b_state_t *blake2b_state;
#if HACL_CAN_COMPILE_SIMD128
Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state;
#endif
#if HACL_CAN_COMPILE_SIMD256
Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state;
#endif
};
blake2_impl impl;
} Blake2Object;

static struct PyMethodDef blake2mod_functions[] = {
{NULL, NULL}
};
#define _Blake2Object_CAST(op) ((Blake2Object *)(op))

// --- BLAKE-2 module clinic configuration ------------------------------------

/*[clinic input]
module _blake2
class _blake2.blake2b "Blake2Object *" "clinic_state()->blake2b_type"
class _blake2.blake2s "Blake2Object *" "clinic_state()->blake2s_type"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7e2b2b3b67a72f18]*/

#define clinic_state() (get_blake2module_state_by_cls(Py_TYPE(self)))
#include "clinic/blake2module.c.h"
#undef clinic_state

// MODULE TYPE SLOTS

static PyType_Spec blake2b_type_spec;
static PyType_Spec blake2s_type_spec;

static int
_blake2_traverse(PyObject *module, visitproc visit, void *arg)
blake2module_traverse(PyObject *module, visitproc visit, void *arg)
{
Blake2State *state = blake2_get_state(module);
blake2module_state *state = get_blake2module_state(module);
Py_VISIT(state->blake2b_type);
Py_VISIT(state->blake2s_type);
return 0;
}

static int
_blake2_clear(PyObject *module)
blake2module_clear(PyObject *module)
{
Blake2State *state = blake2_get_state(module);
blake2module_state *state = get_blake2module_state(module);
Py_CLEAR(state->blake2b_type);
Py_CLEAR(state->blake2s_type);
return 0;
}

static void
_blake2_free(void *module)
blake2module_free(void *module)
{
(void)_blake2_clear((PyObject *)module);
(void)blake2module_clear((PyObject *)module);
}

static void
blake2module_init_cpu_features(Blake2State *state)
blake2module_init_cpu_features(blake2module_state *state)
{
/* This must be kept in sync with hmacmodule_init_cpu_features()
* in hmacmodule.c */
Expand Down Expand Up @@ -202,10 +240,10 @@ blake2module_init_cpu_features(Blake2State *state)
}

static int
blake2_exec(PyObject *m)
blake2module_exec(PyObject *m)
{
Blake2State *st = blake2_get_state(m);
blake2module_init_cpu_features(st);
blake2module_state *state = get_blake2module_state(m);
blake2module_init_cpu_features(state);

#define ADD_INT(DICT, NAME, VALUE) \
do { \
Expand All @@ -229,17 +267,17 @@ blake2_exec(PyObject *m)

ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE);

st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec(
state->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec(
m, &blake2b_type_spec, NULL);

if (st->blake2b_type == NULL) {
if (state->blake2b_type == NULL) {
return -1;
}
if (PyModule_AddType(m, st->blake2b_type) < 0) {
if (PyModule_AddType(m, state->blake2b_type) < 0) {
return -1;
}

PyObject *d = st->blake2b_type->tp_dict;
PyObject *d = state->blake2b_type->tp_dict;
ADD_INT(d, "SALT_SIZE", HACL_HASH_BLAKE2B_SALT_BYTES);
ADD_INT(d, "PERSON_SIZE", HACL_HASH_BLAKE2B_PERSONAL_BYTES);
ADD_INT(d, "MAX_KEY_SIZE", HACL_HASH_BLAKE2B_KEY_BYTES);
Expand All @@ -251,17 +289,17 @@ blake2_exec(PyObject *m)
ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2B_OUT_BYTES);

/* BLAKE2s */
st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec(
state->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec(
m, &blake2s_type_spec, NULL);

if (st->blake2s_type == NULL) {
if (state->blake2s_type == NULL) {
return -1;
}
if (PyModule_AddType(m, st->blake2s_type) < 0) {
if (PyModule_AddType(m, state->blake2s_type) < 0) {
return -1;
}

d = st->blake2s_type->tp_dict;
d = state->blake2s_type->tp_dict;
ADD_INT(d, "SALT_SIZE", HACL_HASH_BLAKE2S_SALT_BYTES);
ADD_INT(d, "PERSON_SIZE", HACL_HASH_BLAKE2S_PERSONAL_BYTES);
ADD_INT(d, "MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES);
Expand All @@ -277,45 +315,35 @@ blake2_exec(PyObject *m)
return 0;
}

static PyModuleDef_Slot _blake2_slots[] = {
{Py_mod_exec, blake2_exec},
static PyModuleDef_Slot blake2module_slots[] = {
{Py_mod_exec, blake2module_exec},
{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
{0, NULL}
};

static struct PyModuleDef blake2_module = {
.m_base = PyModuleDef_HEAD_INIT,
PyDoc_STRVAR(blake2module__doc__,
"_blake2 provides BLAKE2b and BLAKE2s for hashlib\n");

static struct PyModuleDef blake2module_def = {
PyModuleDef_HEAD_INIT,
.m_name = "_blake2",
.m_doc = blake2mod__doc__,
.m_size = sizeof(Blake2State),
.m_methods = blake2mod_functions,
.m_slots = _blake2_slots,
.m_traverse = _blake2_traverse,
.m_clear = _blake2_clear,
.m_free = _blake2_free,
.m_doc = blake2module__doc__,
.m_size = sizeof(blake2module_state),
.m_slots = blake2module_slots,
.m_traverse = blake2module_traverse,
.m_clear = blake2module_clear,
.m_free = blake2module_free,
};

PyMODINIT_FUNC
PyInit__blake2(void)
{
return PyModuleDef_Init(&blake2_module);
return PyModuleDef_Init(&blake2module_def);
}

// IMPLEMENTATION OF METHODS

// The HACL* API does not offer an agile API that can deal with either Blake2S
// or Blake2B -- the reason is that the underlying states are optimized (uint32s
// for S, uint64s for B). Therefore, we use a tagged union in this module to
// correctly dispatch. Note that the previous incarnation of this code
// transformed the Blake2b implementation into the Blake2s one using a script,
// so this is an improvement.
//
// The 128 and 256 versions are only available if i) we were able to compile
// them, and ii) if the CPU we run on also happens to have the right instruction
// set.
typedef enum { Blake2s, Blake2b, Blake2s_128, Blake2b_256 } blake2_impl;

static inline bool
is_blake2b(blake2_impl impl)
{
Expand All @@ -332,52 +360,25 @@ static inline blake2_impl
type_to_impl(PyTypeObject *type)
{
#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256)
Blake2State *st = blake2_get_state_from_type(type);
blake2module_state *state = get_blake2module_state_by_cls(type);
#endif
if (!strcmp(type->tp_name, blake2b_type_spec.name)) {
#if HACL_CAN_COMPILE_SIMD256
return st->can_run_simd256 ? Blake2b_256 : Blake2b;
return state->can_run_simd256 ? Blake2b_256 : Blake2b;
#else
return Blake2b;
#endif
}
else if (!strcmp(type->tp_name, blake2s_type_spec.name)) {
#if HACL_CAN_COMPILE_SIMD128
return st->can_run_simd128 ? Blake2s_128 : Blake2s;
return state->can_run_simd128 ? Blake2s_128 : Blake2s;
#else
return Blake2s;
#endif
}
Py_UNREACHABLE();
}

typedef struct {
HASHLIB_OBJECT_HEAD
union {
Hacl_Hash_Blake2s_state_t *blake2s_state;
Hacl_Hash_Blake2b_state_t *blake2b_state;
#if HACL_CAN_COMPILE_SIMD128
Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state;
#endif
#if HACL_CAN_COMPILE_SIMD256
Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state;
#endif
};
blake2_impl impl;
} Blake2Object;

#define _Blake2Object_CAST(op) ((Blake2Object *)(op))

#include "clinic/blake2module.c.h"

/*[clinic input]
module _blake2
class _blake2.blake2b "Blake2Object *" "&PyBlake2_BLAKE2bType"
class _blake2.blake2s "Blake2Object *" "&PyBlake2_BLAKE2sType"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b7526666bd18af83]*/


static Blake2Object *
new_Blake2Object(PyTypeObject *type)
{
Expand Down
6 changes: 3 additions & 3 deletions Modules/hashlib.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
#include "pycore_lock.h" // PyMutex

/*
* Given a PyObject* obj, fill in the Py_buffer* viewp with the result
* of PyObject_GetBuffer. Sets an exception and issues the erraction
* on any errors, e.g. 'return NULL' or 'goto error'.
* Given a PyObject* 'obj', fill in the Py_buffer* 'viewp' with the result
* of PyObject_GetBuffer. Sets an exception and issues the 'erraction'
* on any errors, e.g., 'return NULL' or 'goto error'.
*/
#define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \
if (PyUnicode_Check((obj))) { \
Expand Down
16 changes: 8 additions & 8 deletions Modules/hmacmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ get_hmacmodule_state_by_cls(PyTypeObject *cls)
return (hmacmodule_state *)state;
}

// --- HMAC Object ------------------------------------------------------------
// --- HMAC object ------------------------------------------------------------

typedef Hacl_Streaming_HMAC_agile_state HACL_HMAC_state;

Expand Down Expand Up @@ -500,11 +500,11 @@ _hacl_hmac_state_update(HACL_HMAC_state *state, uint8_t *buf, Py_ssize_t len)

/* Static information used to construct the hash table. */
static const py_hmac_hinfo py_hmac_static_hinfo[] = {
#define Py_HMAC_HINFO_HACL_API(HACL_HID) \
{ \
/* one-shot helpers */ \
.compute = &Py_hmac_## HACL_HID ##_compute_func, \
.compute_py = &_hmac_compute_## HACL_HID ##_impl, \
#define Py_HMAC_HINFO_HACL_API(HACL_HID) \
{ \
/* one-shot helpers */ \
.compute = &Py_hmac_## HACL_HID ##_compute_func, \
.compute_py = &_hmac_compute_## HACL_HID ##_impl, \
}

#define Py_HMAC_HINFO_ENTRY(HACL_HID, HLIB_NAME) \
Expand Down Expand Up @@ -1683,7 +1683,7 @@ static struct PyModuleDef_Slot hmacmodule_slots[] = {
{0, NULL} /* sentinel */
};

static struct PyModuleDef _hmacmodule = {
static struct PyModuleDef hmacmodule_def = {
PyModuleDef_HEAD_INIT,
.m_name = "_hmac",
.m_size = sizeof(hmacmodule_state),
Expand All @@ -1697,5 +1697,5 @@ static struct PyModuleDef _hmacmodule = {
PyMODINIT_FUNC
PyInit__hmac(void)
{
return PyModuleDef_Init(&_hmacmodule);
return PyModuleDef_Init(&hmacmodule_def);
}
Loading
Loading