Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e7295a8

Browse files
picnixzpicnixz
authored
gh-135239: simpler use of mutexes in cryptographic modules (#135267)
1 parent ac9d37c commit e7295a8

9 files changed

Lines changed: 322 additions & 581 deletions

File tree

‎Lib/test/test_hashlib.py‎

Lines changed: 48 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,57 +1043,78 @@ def test_gil(self):
10431043

10441044
def test_sha256_gil(self):
10451045
gil_minsize = hashlib_helper.find_gil_minsize(['_sha2', '_hashlib'])
1046+
data = b'1' + b'#' * gil_minsize + b'1'
1047+
expected = hashlib.sha256(data).hexdigest()
1048+
10461049
m = hashlib.sha256()
10471050
m.update(b'1')
10481051
m.update(b'#' * gil_minsize)
10491052
m.update(b'1')
1050-
self.assertEqual(
1051-
m.hexdigest(),
1052-
'1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
1053-
)
1053+
self.assertEqual(m.hexdigest(), expected)
10541054

1055-
m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1')
1056-
self.assertEqual(
1057-
m.hexdigest(),
1058-
'1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
1059-
)
1055+
@threading_helper.reap_threads
1056+
@threading_helper.requires_working_threading()
1057+
def test_threaded_hashing_fast(self):
1058+
# Same as test_threaded_hashing_slow() but only tests some functions
1059+
# since otherwise test_hashlib.py becomes too slow during development.
1060+
for name in ['md5', 'sha1', 'sha256', 'sha3_256', 'blake2s']:
1061+
if constructor := getattr(hashlib, name, None):
1062+
with self.subTest(name):
1063+
self.do_test_threaded_hashing(constructor, is_shake=False)
1064+
if shake_128 := getattr(hashlib, 'shake_128', None):
1065+
self.do_test_threaded_hashing(shake_128, is_shake=True)
10601066

1067+
@requires_resource('cpu')
10611068
@threading_helper.reap_threads
10621069
@threading_helper.requires_working_threading()
1063-
def test_threaded_hashing(self):
1070+
def test_threaded_hashing_slow(self):
1071+
for algorithm, constructors in self.constructors_to_test.items():
1072+
is_shake = algorithm in self.shakes
1073+
for constructor in constructors:
1074+
with self.subTest(constructor.__name__, is_shake=is_shake):
1075+
self.do_test_threaded_hashing(constructor, is_shake)
1076+
1077+
def do_test_threaded_hashing(self, constructor, is_shake):
10641078
# Updating the same hash object from several threads at once
10651079
# using data chunk sizes containing the same byte sequences.
10661080
#
10671081
# If the internal locks are working to prevent multiple
10681082
# updates on the same object from running at once, the resulting
10691083
# hash will be the same as doing it single threaded upfront.
1070-
hasher = hashlib.sha1()
1071-
num_threads = 5
1072-
smallest_data = b'swineflu'
1073-
data = smallest_data * 200000
1074-
expected_hash = hashlib.sha1(data*num_threads).hexdigest()
1075-
1076-
def hash_in_chunks(chunk_size):
1077-
index = 0
1078-
while index < len(data):
1079-
hasher.update(data[index:index + chunk_size])
1080-
index += chunk_size
1084+
1085+
# The data to hash has length s|M|q^N and the chunk size for the i-th
1086+
# thread is s|M|q^(N-i), where N is the number of threads, M is a fixed
1087+
# message of small length, and s >= 1 and q >= 2 are small integers.
1088+
smallest_size, num_threads, s, q = 8, 5, 2, 10
1089+
1090+
smallest_data = os.urandom(smallest_size)
1091+
data = s * smallest_data * (q ** num_threads)
1092+
1093+
h1 = constructor(usedforsecurity=False)
1094+
h2 = constructor(data * num_threads, usedforsecurity=False)
1095+
1096+
def update(chunk_size):
1097+
for index in range(0, len(data), chunk_size):
1098+
h1.update(data[index:index + chunk_size])
10811099

10821100
threads = []
1083-
for threadnum in range(num_threads):
1084-
chunk_size = len(data) // (10 ** threadnum)
1101+
for thread_num in range(num_threads):
1102+
# chunk_size = len(data) // (q ** thread_num)
1103+
chunk_size = s * smallest_size * q ** (num_threads - thread_num)
10851104
self.assertGreater(chunk_size, 0)
1086-
self.assertEqual(chunk_size % len(smallest_data), 0)
1087-
thread = threading.Thread(target=hash_in_chunks,
1088-
args=(chunk_size,))
1105+
self.assertEqual(chunk_size % smallest_size, 0)
1106+
thread = threading.Thread(target=update, args=(chunk_size,))
10891107
threads.append(thread)
10901108

10911109
for thread in threads:
10921110
thread.start()
10931111
for thread in threads:
10941112
thread.join()
10951113

1096-
self.assertEqual(expected_hash, hasher.hexdigest())
1114+
if is_shake:
1115+
self.assertEqual(h1.hexdigest(16), h2.hexdigest(16))
1116+
else:
1117+
self.assertEqual(h1.hexdigest(), h2.hexdigest())
10971118

10981119
def test_get_fips_mode(self):
10991120
fips_mode = self.is_fips_mode

‎Modules/_hashopenssl.c‎

Lines changed: 23 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -278,21 +278,15 @@ get_hashlib_state(PyObject *module)
278278
}
279279

280280
typedef struct {
281-
PyObject_HEAD
281+
HASHLIB_OBJECT_HEAD
282282
EVP_MD_CTX *ctx; /* OpenSSL message digest context */
283-
// Prevents undefined behavior via multiple threads entering the C API.
284-
bool use_mutex;
285-
PyMutex mutex; /* OpenSSL context lock */
286283
} HASHobject;
287284

288285
#define HASHobject_CAST(op) ((HASHobject *)(op))
289286

290287
typedef struct {
291-
PyObject_HEAD
288+
HASHLIB_OBJECT_HEAD
292289
HMAC_CTX *ctx; /* OpenSSL hmac context */
293-
// Prevents undefined behavior via multiple threads entering the C API.
294-
bool use_mutex;
295-
PyMutex mutex; /* HMAC context lock */
296290
} HMACobject;
297291

298292
#define HMACobject_CAST(op) ((HMACobject *)(op))
@@ -700,9 +694,9 @@ static int
700694
_hashlib_HASH_copy_locked(HASHobject *self, EVP_MD_CTX *new_ctx_p)
701695
{
702696
int result;
703-
ENTER_HASHLIB(self);
697+
HASHLIB_ACQUIRE_LOCK(self);
704698
result = EVP_MD_CTX_copy(new_ctx_p, self->ctx);
705-
LEAVE_HASHLIB(self);
699+
HASHLIB_RELEASE_LOCK(self);
706700
if (result == 0) {
707701
notify_smart_ssl_error_occurred_in(Py_STRINGIFY(EVP_MD_CTX_copy));
708702
return -1;
@@ -802,27 +796,13 @@ _hashlib_HASH_update_impl(HASHobject *self, PyObject *obj)
802796
{
803797
int result;
804798
Py_buffer view;
805-
806799
GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
807-
808-
if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
809-
self->use_mutex = true;
810-
}
811-
if (self->use_mutex) {
812-
Py_BEGIN_ALLOW_THREADS
813-
PyMutex_Lock(&self->mutex);
814-
result = _hashlib_HASH_hash(self, view.buf, view.len);
815-
PyMutex_Unlock(&self->mutex);
816-
Py_END_ALLOW_THREADS
817-
} else {
818-
result = _hashlib_HASH_hash(self, view.buf, view.len);
819-
}
820-
800+
HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(
801+
self, view.len,
802+
result = _hashlib_HASH_hash(self, view.buf, view.len)
803+
);
821804
PyBuffer_Release(&view);
822-
823-
if (result == -1)
824-
return NULL;
825-
Py_RETURN_NONE;
805+
return result < 0 ? NULL : Py_None;
826806
}
827807

828808
static PyMethodDef HASH_methods[] = {
@@ -1144,15 +1124,12 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj,
11441124
}
11451125

11461126
if (view.buf && view.len) {
1147-
if (view.len >= HASHLIB_GIL_MINSIZE) {
1148-
/* We do not initialize self->lock here as this is the constructor
1149-
* where it is not yet possible to have concurrent access. */
1150-
Py_BEGIN_ALLOW_THREADS
1151-
result = _hashlib_HASH_hash(self, view.buf, view.len);
1152-
Py_END_ALLOW_THREADS
1153-
} else {
1154-
result = _hashlib_HASH_hash(self, view.buf, view.len);
1155-
}
1127+
/* Do not use self->mutex here as this is the constructor
1128+
* where it is not yet possible to have concurrent access. */
1129+
HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(
1130+
view.len,
1131+
result = _hashlib_HASH_hash(self, view.buf, view.len)
1132+
);
11561133
if (result == -1) {
11571134
assert(PyErr_Occurred());
11581135
Py_CLEAR(self);
@@ -1813,9 +1790,9 @@ static int
18131790
locked_HMAC_CTX_copy(HMAC_CTX *new_ctx_p, HMACobject *self)
18141791
{
18151792
int result;
1816-
ENTER_HASHLIB(self);
1793+
HASHLIB_ACQUIRE_LOCK(self);
18171794
result = HMAC_CTX_copy(new_ctx_p, self->ctx);
1818-
LEAVE_HASHLIB(self);
1795+
HASHLIB_RELEASE_LOCK(self);
18191796
if (result == 0) {
18201797
notify_smart_ssl_error_occurred_in(Py_STRINGIFY(HMAC_CTX_copy));
18211798
return -1;
@@ -1846,24 +1823,12 @@ _hmac_update(HMACobject *self, PyObject *obj)
18461823
Py_buffer view = {0};
18471824

18481825
GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0);
1849-
1850-
if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
1851-
self->use_mutex = true;
1852-
}
1853-
if (self->use_mutex) {
1854-
Py_BEGIN_ALLOW_THREADS
1855-
PyMutex_Lock(&self->mutex);
1856-
r = HMAC_Update(self->ctx,
1857-
(const unsigned char *)view.buf,
1858-
(size_t)view.len);
1859-
PyMutex_Unlock(&self->mutex);
1860-
Py_END_ALLOW_THREADS
1861-
} else {
1862-
r = HMAC_Update(self->ctx,
1863-
(const unsigned char *)view.buf,
1864-
(size_t)view.len);
1865-
}
1866-
1826+
HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(
1827+
self, view.len,
1828+
r = HMAC_Update(
1829+
self->ctx, (const unsigned char *)view.buf, (size_t)view.len
1830+
)
1831+
);
18671832
PyBuffer_Release(&view);
18681833

18691834
if (r == 0) {

‎Modules/blake2module.c‎

Lines changed: 20 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ type_to_impl(PyTypeObject *type)
352352
}
353353

354354
typedef struct {
355-
PyObject_HEAD
355+
HASHLIB_OBJECT_HEAD
356356
union {
357357
Hacl_Hash_Blake2s_state_t *blake2s_state;
358358
Hacl_Hash_Blake2b_state_t *blake2b_state;
@@ -364,8 +364,6 @@ typedef struct {
364364
#endif
365365
};
366366
blake2_impl impl;
367-
bool use_mutex;
368-
PyMutex mutex;
369367
} Blake2Object;
370368

371369
#define _Blake2Object_CAST(op) ((Blake2Object *)(op))
@@ -422,7 +420,7 @@ new_Blake2Object(PyTypeObject *type)
422420
} while (0)
423421

424422
static void
425-
update(Blake2Object *self, uint8_t *buf, Py_ssize_t len)
423+
blake2_update_unlocked(Blake2Object *self, uint8_t *buf, Py_ssize_t len)
426424
{
427425
switch (self->impl) {
428426
// blake2b_256_state and blake2s_128_state must be if'd since
@@ -646,14 +644,12 @@ py_blake2_new(PyTypeObject *type, PyObject *data, int digest_size,
646644
if (data != NULL) {
647645
Py_buffer buf;
648646
GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error);
649-
if (buf.len >= HASHLIB_GIL_MINSIZE) {
650-
Py_BEGIN_ALLOW_THREADS
651-
update(self, buf.buf, buf.len);
652-
Py_END_ALLOW_THREADS
653-
}
654-
else {
655-
update(self, buf.buf, buf.len);
656-
}
647+
/* Do not use self->mutex here as this is the constructor
648+
* where it is not yet possible to have concurrent access. */
649+
HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(
650+
buf.len,
651+
blake2_update_unlocked(self, buf.buf, buf.len)
652+
);
657653
PyBuffer_Release(&buf);
658654
}
659655

@@ -744,7 +740,7 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size,
744740
}
745741

746742
static int
747-
blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy)
743+
blake2_blake2b_copy_unlocked(Blake2Object *self, Blake2Object *cpy)
748744
{
749745
assert(cpy != NULL);
750746
#define BLAKE2_COPY(TYPE, STATE_ATTR) \
@@ -801,9 +797,9 @@ _blake2_blake2b_copy_impl(Blake2Object *self)
801797
return NULL;
802798
}
803799

804-
ENTER_HASHLIB(self);
805-
rc = blake2_blake2b_copy_locked(self, cpy);
806-
LEAVE_HASHLIB(self);
800+
HASHLIB_ACQUIRE_LOCK(self);
801+
rc = blake2_blake2b_copy_unlocked(self, cpy);
802+
HASHLIB_RELEASE_LOCK(self);
807803
if (rc < 0) {
808804
Py_DECREF(cpy);
809805
return NULL;
@@ -825,25 +821,12 @@ _blake2_blake2b_update_impl(Blake2Object *self, PyObject *data)
825821
/*[clinic end generated code: output=99330230068e8c99 input=ffc4aa6a6a225d31]*/
826822
{
827823
Py_buffer buf;
828-
829824
GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
830-
831-
if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
832-
self->use_mutex = true;
833-
}
834-
if (self->use_mutex) {
835-
Py_BEGIN_ALLOW_THREADS
836-
PyMutex_Lock(&self->mutex);
837-
update(self, buf.buf, buf.len);
838-
PyMutex_Unlock(&self->mutex);
839-
Py_END_ALLOW_THREADS
840-
}
841-
else {
842-
update(self, buf.buf, buf.len);
843-
}
844-
825+
HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(
826+
self, buf.len,
827+
blake2_update_unlocked(self, buf.buf, buf.len)
828+
);
845829
PyBuffer_Release(&buf);
846-
847830
Py_RETURN_NONE;
848831
}
849832

@@ -881,9 +864,9 @@ _blake2_blake2b_digest_impl(Blake2Object *self)
881864
/*[clinic end generated code: output=31ab8ad477f4a2f7 input=7d21659e9c5fff02]*/
882865
{
883866
uint8_t digest_length = 0, digest[HACL_HASH_BLAKE2B_OUT_BYTES];
884-
ENTER_HASHLIB(self);
867+
HASHLIB_ACQUIRE_LOCK(self);
885868
digest_length = blake2_blake2b_compute_digest(self, digest);
886-
LEAVE_HASHLIB(self);
869+
HASHLIB_RELEASE_LOCK(self);
887870
return PyBytes_FromStringAndSize((const char *)digest, digest_length);
888871
}
889872

@@ -898,9 +881,9 @@ _blake2_blake2b_hexdigest_impl(Blake2Object *self)
898881
/*[clinic end generated code: output=5ef54b138db6610a input=76930f6946351f56]*/
899882
{
900883
uint8_t digest_length = 0, digest[HACL_HASH_BLAKE2B_OUT_BYTES];
901-
ENTER_HASHLIB(self);
884+
HASHLIB_ACQUIRE_LOCK(self);
902885
digest_length = blake2_blake2b_compute_digest(self, digest);
903-
LEAVE_HASHLIB(self);
886+
HASHLIB_RELEASE_LOCK(self);
904887
return _Py_strhex((const char *)digest, digest_length);
905888
}
906889

0 commit comments

Comments
 (0)