Merge pull request #23898 from seberg/string-nonzero-sanity

charris · web-flow · commit 19f86c31854e · 2023-06-09T17:42:34.000-06:00
API: Change string to bool conversions to be consistent with Python
diff --git a/doc/release/upcoming_changes/23871.change.rst b/doc/release/upcoming_changes/23871.change.rst
@@ -0,0 +1,26 @@
+Truthyness of NumPy strings changed
+-----------------------------------
+NumPy strings previously were inconsistent about how they defined
+if the string is ``True`` or ``False`` and the definition did not
+match the one used by Python.
+Strings are now considered ``True`` when they are non-empty and
+``False`` when they are empty.
+This changes the following distinct cases:
+
+* Casts from string to boolean were previously roughly equivalent
+  to ``string_array.astype(np.int64).astype(bool)``, meaning that only
+  valid integers could be cast.
+  Now a string of ``"0"`` will be considered ``True`` since it is not empty.
+  If you need the old behavior, you may use the above step (casting
+  to integer first) or ``string_array == "0"`` (if the input is only ever ``0`` or ``1``).
+  To get the new result on old NumPy versions use ``string_array != ""``.
+* ``np.nonzero(string_array)`` previously ignored whitespace so that
+  a string only containing whitepsace was considered ``False``.
+  Whitespace is now considered ``True``.
+
+This change does not affect ``np.loadtxt``, ``np.fromstring``, or ``np.genfromtxt``.
+The first two still use the integer definition, while ``genfromtxt`` continues to
+match for ``"true"`` (ignoring case).
+However, if ``np.bool_`` is used as a converter the result will change.
+
+The change does affect ``np.fromregex`` as it uses direct assignments.
diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src
@@ -1771,13 +1771,6 @@ static void
         if (temp == NULL) {
             return;
         }
-#if @is_string_to_bool@
-        /* Legacy behaviour converts strings to integers before going to bool */
-        Py_SETREF(temp, PyNumber_Long(temp));
-        if (temp == NULL) {
-            return;
-        }
-#endif
         if (@to@_setitem(temp, op, aop)) {
             Py_DECREF(temp);
             return;
@@ -2802,79 +2795,45 @@ static npy_bool
 /**end repeat**/
 
 
-#define WHITESPACE " \t\n\r\v\f"
-#define WHITELEN 6
-
-static npy_bool
-Py_STRING_ISSPACE(char ch)
-{
-    char white[] = WHITESPACE;
-    int j;
-    npy_bool space = NPY_FALSE;
-
-    for (j = 0; j < WHITELEN; j++) {
-        if (ch == white[j]) {
-            space = NPY_TRUE;
-            break;
-        }
-    }
-    return space;
-}
 
 static npy_bool
 STRING_nonzero (char *ip, PyArrayObject *ap)
 {
     int len = PyArray_DESCR(ap)->elsize;
-    int i;
-    npy_bool nonz = NPY_FALSE;
-    npy_bool seen_null = NPY_FALSE;
 
-    for (i = 0; i < len; i++) {
-        if (*ip == '\0') {
-            seen_null = NPY_TRUE;
+    for (int i = 0; i < len; i++) {
+        if (ip[i]) {
+            return NPY_TRUE;
         }
-        else if (seen_null || !Py_STRING_ISSPACE(*ip)) {
-            nonz = NPY_TRUE;
-            break;
-        }
-        ip++;
     }
-    return nonz;
+
+    return NPY_FALSE;
 }
 
 static npy_bool
-UNICODE_nonzero (npy_ucs4 *ip, PyArrayObject *ap)
+UNICODE_nonzero (char *ip, PyArrayObject *ap)
 {
-    int len = PyArray_DESCR(ap)->elsize >> 2;
-    int i;
-    npy_bool nonz = NPY_FALSE;
-    npy_bool seen_null = NPY_FALSE;
-    char *buffer = NULL;
-
-    if (PyArray_ISBYTESWAPPED(ap) || !PyArray_ISALIGNED(ap)) {
-        buffer = PyArray_malloc(PyArray_DESCR(ap)->elsize);
-        if (buffer == NULL) {
-            return nonz;
-        }
-        memcpy(buffer, ip, PyArray_DESCR(ap)->elsize);
-        if (PyArray_ISBYTESWAPPED(ap)) {
-            byte_swap_vector(buffer, len, 4);
+    if (PyArray_ISALIGNED(ap)) {
+        /* go character by character */
+        Py_UCS4 *chars = (Py_UCS4 *)ip;
+        int len = PyArray_DESCR(ap)->elsize / 4;
+        for (int i = 0; i < len; i++) {
+            if (chars[i]) {
+                return NPY_TRUE;
+            }
         }
-        ip = (npy_ucs4 *)buffer;
     }
-
-    for (i = 0; i < len; i++) {
-        if (*ip == '\0') {
-            seen_null = NPY_TRUE;
-        }
-        else if (seen_null || !Py_UNICODE_ISSPACE(*ip)) {
-            nonz = NPY_TRUE;
-            break;
+    else {
+        /* go char/byte by char/byte, it doesn't matter where the nonzero is */
+        int len = PyArray_DESCR(ap)->elsize;
+        for (int i = 0; i < len; i++) {
+            if (ip[i]) {
+                return NPY_TRUE;
+            }
         }
-        ip++;
     }
-    PyArray_free(buffer);
-    return nonz;
+
+    return NPY_FALSE;
 }
 
 static npy_bool
diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py
@@ -321,30 +321,21 @@ def test_array_astype_warning(t):
 @pytest.mark.parametrize(["dtype", "out_dtype"],
         [(np.bytes_, np.bool_),
          (np.str_, np.bool_),
-         (np.dtype("S10,S9"), np.dtype("?,?"))])
+         (np.dtype("S10,S9"), np.dtype("?,?")),
+         # The following also checks unaligned unicode access:
+         (np.dtype("S7,U9"), np.dtype("?,?"))])
 def test_string_to_boolean_cast(dtype, out_dtype):
-    """
-    Currently, for `astype` strings are cast to booleans effectively by
-    calling `bool(int(string)`. This is not consistent (see gh-9875) and
-    will eventually be deprecated.
-    """
-    arr = np.array(["10", "10\0\0\0", "0\0\0", "0"], dtype=dtype)
-    expected = np.array([True, True, False, False], dtype=out_dtype)
+    # Only the last two (empty) strings are falsy (the `\0` is stripped):
+    arr = np.array(
+            ["10", "10\0\0\0", "0\0\0", "0", "False", " ", "", "\0"],
+            dtype=dtype)
+    expected = np.array(
+            [True, True, True, True, True, True, False, False],
+            dtype=out_dtype)
     assert_array_equal(arr.astype(out_dtype), expected)
-
-@pytest.mark.parametrize(["dtype", "out_dtype"],
-        [(np.bytes_, np.bool_),
-         (np.str_, np.bool_),
-         (np.dtype("S10,S9"), np.dtype("?,?"))])
-def test_string_to_boolean_cast_errors(dtype, out_dtype):
-    """
-    These currently error out, since cast to integers fails, but should not
-    error out in the future.
-    """
-    for invalid in ["False", "True", "", "\0", "non-empty"]:
-        arr = np.array([invalid], dtype=dtype)
-        with assert_raises(ValueError):
-            arr.astype(out_dtype)
+    # As it's similar, check that nonzero behaves the same (structs are
+    # nonzero if all entries are)
+    assert_array_equal(np.nonzero(arr), np.nonzero(expected))
 
 @pytest.mark.parametrize("str_type", [str, bytes, np.str_, np.unicode_])
 @pytest.mark.parametrize("scalar_type",
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
@@ -9117,10 +9117,10 @@ class TestBytestringArrayNonzero:
     def test_empty_bstring_array_is_falsey(self):
         assert_(not np.array([''], dtype=str))
 
-    def test_whitespace_bstring_array_is_falsey(self):
+    def test_whitespace_bstring_array_is_truthy(self):
         a = np.array(['spam'], dtype=str)
         a[0] = '  \0\0'
-        assert_(not a)
+        assert_(a)
 
     def test_all_null_bstring_array_is_falsey(self):
         a = np.array(['spam'], dtype=str)
@@ -9166,10 +9166,10 @@ class TestUnicodeArrayNonzero:
     def test_empty_ustring_array_is_falsey(self):
         assert_(not np.array([''], dtype=np.str_))
 
-    def test_whitespace_ustring_array_is_falsey(self):
+    def test_whitespace_ustring_array_is_truthy(self):
         a = np.array(['eggs'], dtype=np.str_)
         a[0] = '  \0\0'
-        assert_(not a)
+        assert_(a)
 
     def test_all_null_ustring_array_is_falsey(self):
         a = np.array(['eggs'], dtype=np.str_)
