Thanks to visit codestin.com
Credit goes to github.com

Skip to content

BUG: fix choose refcount leak #24188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 31, 2023
Merged

BUG: fix choose refcount leak #24188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2f92f29
BUG: fix choose refcount leak
tylerjereddy Jul 15, 2023
c4261af
BUG: PR 24188 revisions
tylerjereddy Jul 18, 2023
68eead8
MAINT: PR 24188 revisions
tylerjereddy Jul 25, 2023
fd96b34
MAINT: Use explicit copy path in choose based on refcheck
seberg Jul 31, 2023
74979f3
BUG: Remove unnecessary (and now also segfaulting) iterator creation
seberg Jul 31, 2023
c176234
ENH: Use `memcpy` and swap order
seberg Jul 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions numpy/core/src/multiarray/item_selection.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -993,9 +993,10 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out,
if (multi == NULL) {
goto fail;
}
dtype = PyArray_DESCR(mps[0]);

/* Set-up return array */
if (out == NULL) {
dtype = PyArray_DESCR(mps[0]);
Py_INCREF(dtype);
obj = (PyArrayObject *)PyArray_NewFromDescr(Py_TYPE(ap),
dtype,
Expand Down Expand Up @@ -1032,16 +1033,29 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out,
*/
flags |= NPY_ARRAY_ENSURECOPY;
}
dtype = PyArray_DESCR(mps[0]);
Py_INCREF(dtype);
obj = (PyArrayObject *)PyArray_FromArray(out, dtype, flags);
}

if (obj == NULL) {
goto fail;
}
elsize = PyArray_DESCR(obj)->elsize;
elsize = dtype->elsize;
ret_data = PyArray_DATA(obj);
npy_intp transfer_strides[2] = {elsize, elsize};
npy_intp one = 1;
NPY_ARRAYMETHOD_FLAGS transfer_flags = 0;
NPY_cast_info cast_info = {.func = NULL};
if (PyDataType_REFCHK(dtype)) {
int is_aligned = IsUintAligned(obj);
PyArray_GetDTypeTransferFunction(
is_aligned,
dtype->elsize,
dtype->elsize,
dtype,
dtype, 0, &cast_info,
&transfer_flags);
}

while (PyArray_MultiIter_NOTDONE(multi)) {
mi = *((npy_intp *)PyArray_MultiIter_DATA(multi, n));
Expand Down Expand Up @@ -1074,12 +1088,22 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out,
break;
}
}
memmove(ret_data, PyArray_MultiIter_DATA(multi, mi), elsize);
if (cast_info.func == NULL) {
/* We ensure memory doesn't overlap, so can use memcpy */
memcpy(ret_data, PyArray_MultiIter_DATA(multi, mi), elsize);
}
else {
char *args[2] = {PyArray_MultiIter_DATA(multi, mi), ret_data};
if (cast_info.func(&cast_info.context, args, &one,
transfer_strides, cast_info.auxdata) < 0) {
goto fail;
}
}
ret_data += elsize;
PyArray_MultiIter_NEXT(multi);
}

PyArray_INCREF(obj);
NPY_cast_info_xfree(&cast_info);
Py_DECREF(multi);
for (i = 0; i < n; i++) {
Py_XDECREF(mps[i]);
Expand All @@ -1095,6 +1119,7 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out,
return (PyObject *)obj;

fail:
NPY_cast_info_xfree(&cast_info);
Py_XDECREF(multi);
for (i = 0; i < n; i++) {
Py_XDECREF(mps[i]);
Expand Down
11 changes: 11 additions & 0 deletions numpy/core/tests/test_multiarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10028,3 +10028,14 @@ def test_argsort_int(N, dtype):
arr = rnd.randint(low=minv, high=maxv, size=N, dtype=dtype)
arr[N-1] = maxv
assert_arg_sorted(arr, np.argsort(arr, kind='quick'))


@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
def test_gh_22683():
b = 777.68760986
a = np.array([b] * 10000, dtype=object)
refc_start = sys.getrefcount(b)
np.choose(np.zeros(10000, dtype=int), [a], out=a)
np.choose(np.zeros(10000, dtype=int), [a], out=a)
refc_end = sys.getrefcount(b)
assert refc_end - refc_start < 10
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could probably go less than 10 here, though I think each incantation can at least do +1

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In case you iterate: Randomly realized that this actually stopped doing anything useful on Python 3.12, because IIRC the literal 1 will be an immortal object (and thus it's reference count doesn't change reliably).