1.16 maintenance branch fails tests on MSVC 2015 (VS2015 update 2) #12520

eric-wieser · 2018-12-09T21:48:15Z

On my windows 10 machine, I get a lot of NaN-based failures, with:

Python: 3.5.4 (v3.5.4:3f56838, Aug 8 2017, 02:17:05) [MSC v.1900 64 bit (AMD64)]
Numpy: a clean checkout of f07a38d

Build logs

================================== FAILURES ===================================
_________________________ TestMaximum.test_float_nans _________________________

self = <numpy.core.tests.test_umath.TestMaximum object at 0x000001EE0505FF28>

    def test_float_nans(self):
        nan = np.nan
        arg1 = np.array([0,   nan, nan])
        arg2 = np.array([nan, 0,   nan])
        out = np.array([nan, nan, nan])
>       assert_equal(np.maximum(arg1, arg2), out)

arg1       = array([ 0., nan, nan])
arg2       = array([nan,  0., nan])
nan        = nan
out        = array([nan, nan, nan])
self       = <numpy.core.tests.test_umath.TestMaximum object at 0x000001EE0505FF28>

numpy\core\tests\test_umath.py:908:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

x = array([ 0., nan, nan]), y = array([nan, nan, nan]), func = <ufunc 'isnan'>
hasval = 'nan'

    def func_assert_same_pos(x, y, func=isnan, hasval='nan'):
        """Handling nan/inf.

            Combine results of running func on x and y, checking that they are True
            at the same locations.

            """
        x_id = func(x)
        y_id = func(y)
        # We include work-arounds here to handle three types of slightly
        # pathological ndarray subclasses:
        # (1) all() on `masked` array scalars can return masked arrays, so we
        #     use != True
        # (2) __eq__ on some ndarray subclasses returns Python booleans
        #     instead of element-wise comparisons, so we cast to bool_() and
        #     use isinstance(..., bool) checks
        # (3) subclasses with bare-bones __array_function__ implemenations may
        #     not implement np.all(), so favor using the .all() method
        # We are not committed to supporting such subclasses, but it's nice to
        # support them if possible.
        if bool_(x_id == y_id).all() != True:
            msg = build_err_msg([x, y],
                                err_msg + '\nx and y %s location mismatch:'
                                % (hasval), verbose=verbose, header=header,
                                names=('x', 'y'), precision=precision)
>           raise AssertionError(msg)
E           AssertionError:
E           Arrays are not equal
E
E           x and y nan location mismatch:
E            x: array([ 0., nan, nan])
E            y: array([nan, nan, nan])

bool_      = <class 'numpy.bool_'>
err_msg    = ''
func       = <ufunc 'isnan'>
hasval     = 'nan'
header     = 'Arrays are not equal'
msg        = '\nArrays are not equal\n\nx and y nan location mismatch:\n x: array([ 0., nan, nan])\n y: array([nan, nan, nan])'
precision  = 6
verbose    = True
x          = array([ 0., nan, nan])
x_id       = array([False,  True,  True])
y          = array([nan, nan, nan])
y_id       = array([ True,  True,  True])

numpy\testing\_private\utils.py:728: AssertionError
_________________________ TestMinimum.test_float_nans _________________________

self = <numpy.core.tests.test_umath.TestMinimum object at 0x000001EE050434A8>

    def test_float_nans(self):
        nan = np.nan
        arg1 = np.array([0,   nan, nan])
        arg2 = np.array([nan, 0,   nan])
        out = np.array([nan, nan, nan])
>       assert_equal(np.minimum(arg1, arg2), out)

arg1       = array([ 0., nan, nan])
arg2       = array([nan,  0., nan])
nan        = nan
out        = array([nan, nan, nan])
self       = <numpy.core.tests.test_umath.TestMinimum object at 0x000001EE050434A8>

numpy\core\tests\test_umath.py:966:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

x = array([ 0., nan, nan]), y = array([nan, nan, nan]), func = <ufunc 'isnan'>
hasval = 'nan'

    def func_assert_same_pos(x, y, func=isnan, hasval='nan'):
        """Handling nan/inf.

            Combine results of running func on x and y, checking that they are True
            at the same locations.

            """
        x_id = func(x)
        y_id = func(y)
        # We include work-arounds here to handle three types of slightly
        # pathological ndarray subclasses:
        # (1) all() on `masked` array scalars can return masked arrays, so we
        #     use != True
        # (2) __eq__ on some ndarray subclasses returns Python booleans
        #     instead of element-wise comparisons, so we cast to bool_() and
        #     use isinstance(..., bool) checks
        # (3) subclasses with bare-bones __array_function__ implemenations may
        #     not implement np.all(), so favor using the .all() method
        # We are not committed to supporting such subclasses, but it's nice to
        # support them if possible.
        if bool_(x_id == y_id).all() != True:
            msg = build_err_msg([x, y],
                                err_msg + '\nx and y %s location mismatch:'
                                % (hasval), verbose=verbose, header=header,
                                names=('x', 'y'), precision=precision)
>           raise AssertionError(msg)
E           AssertionError:
E           Arrays are not equal
E
E           x and y nan location mismatch:
E            x: array([ 0., nan, nan])
E            y: array([nan, nan, nan])

bool_      = <class 'numpy.bool_'>
err_msg    = ''
func       = <ufunc 'isnan'>
hasval     = 'nan'
header     = 'Arrays are not equal'
msg        = '\nArrays are not equal\n\nx and y nan location mismatch:\n x: array([ 0., nan, nan])\n y: array([nan, nan, nan])'
precision  = 6
verbose    = True
x          = array([ 0., nan, nan])
x_id       = array([False,  True,  True])
y          = array([nan, nan, nan])
y_id       = array([ True,  True,  True])

numpy\testing\_private\utils.py:728: AssertionError
____________________________ TestFmax.test_reduce _____________________________

self = <numpy.core.tests.test_umath.TestFmax object at 0x000001EE01EC3780>

    def test_reduce(self):
        dflt = np.typecodes['AllFloat']
        dint = np.typecodes['AllInteger']
        seq1 = np.arange(11)
        seq2 = seq1[::-1]
        func = np.fmax.reduce
        for dt in dint:
            tmp1 = seq1.astype(dt)
            tmp2 = seq2.astype(dt)
            assert_equal(func(tmp1), 10)
            assert_equal(func(tmp2), 10)
        for dt in dflt:
            tmp1 = seq1.astype(dt)
            tmp2 = seq2.astype(dt)
            assert_equal(func(tmp1), 10)
            assert_equal(func(tmp2), 10)
            tmp1[::2] = np.nan
            tmp2[::2] = np.nan
>           assert_equal(func(tmp1), 9)
E           AssertionError:
E           Items are not equal:
E            ACTUAL: nan
E            DESIRED: 9

dflt       = 'efdgFDG'
dint       = 'bBhHiIlLqQpP'
dt         = 'f'
func       = <built-in method reduce of numpy.ufunc object at 0x000001EE76ADBB40>
self       = <numpy.core.tests.test_umath.TestFmax object at 0x000001EE01EC3780>
seq1       = array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10])
seq2       = array([10,  9,  8,  7,  6,  5,  4,  3,  2,  1,  0])
tmp1       = array([nan,  1., nan,  3., nan,  5., nan,  7., nan,  9., nan],
      dtype=float32)
tmp2       = array([nan,  9., nan,  7., nan,  5., nan,  3., nan,  1., nan],
      dtype=float32)

numpy\core\tests\test_umath.py:1012: AssertionError
____________________________ TestFmin.test_reduce _____________________________

self = <numpy.core.tests.test_umath.TestFmin object at 0x000001EE01ECAB38>

    def test_reduce(self):
        dflt = np.typecodes['AllFloat']
        dint = np.typecodes['AllInteger']
        seq1 = np.arange(11)
        seq2 = seq1[::-1]
        func = np.fmin.reduce
        for dt in dint:
            tmp1 = seq1.astype(dt)
            tmp2 = seq2.astype(dt)
            assert_equal(func(tmp1), 0)
            assert_equal(func(tmp2), 0)
        for dt in dflt:
            tmp1 = seq1.astype(dt)
            tmp2 = seq2.astype(dt)
            assert_equal(func(tmp1), 0)
            assert_equal(func(tmp2), 0)
            tmp1[::2] = np.nan
            tmp2[::2] = np.nan
>           assert_equal(func(tmp1), 1)
E           AssertionError:
E           Items are not equal:
E            ACTUAL: nan
E            DESIRED: 1

dflt       = 'efdgFDG'
dint       = 'bBhHiIlLqQpP'
dt         = 'f'
func       = <built-in method reduce of numpy.ufunc object at 0x000001EE76ADBD00>
self       = <numpy.core.tests.test_umath.TestFmin object at 0x000001EE01ECAB38>
seq1       = array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10])
seq2       = array([10,  9,  8,  7,  6,  5,  4,  3,  2,  1,  0])
tmp1       = array([nan,  1., nan,  3., nan,  5., nan,  7., nan,  9., nan],
      dtype=float32)
tmp2       = array([nan,  9., nan,  7., nan,  5., nan,  3., nan,  1., nan],
      dtype=float32)

numpy\core\tests\test_umath.py:1054: AssertionError
_______________________ TestMinMax.test_minmax_blocked ________________________

self = <numpy.core.tests.test_umath.TestMinMax object at 0x000001EE053C8048>

    def test_minmax_blocked(self):
        # simd tests on max/min, test all alignments, slow but important
        # for 2 * vz + 2 * (vs - 1) + 1 (unrolled once)
        for dt, sz in [(np.float32, 15), (np.float64, 7)]:
            for out, inp, msg in _gen_alignment_data(dtype=dt, type='unary',
                                                     max_size=sz):
                for i in range(inp.size):
                    inp[:] = np.arange(inp.size, dtype=dt)
                    inp[i] = np.nan
                    emsg = lambda: '%r\n%s' % (inp, msg)
                    with suppress_warnings() as sup:
                        sup.filter(RuntimeWarning,
                                   "invalid value encountered in reduce")
>                       assert_(np.isnan(inp.max()), msg=emsg)
E                       AssertionError: array([ 0., nan], dtype=float32)
E                       unary offset=(0, 0), size=2, dtype=<class 'numpy.float32'>, out of place

dt         = <class 'numpy.float32'>
emsg       = <function TestMinMax.test_minmax_blocked.<locals>.<lambda> at 0x000001EE041E9730>
i          = 1
inp        = array([ 0., nan], dtype=float32)
msg        = "unary offset=(0, 0), size=2, dtype=<class 'numpy.float32'>, out of place"
out        = array([ 0., nan], dtype=float32)
self       = <numpy.core.tests.test_umath.TestMinMax object at 0x000001EE053C8048>
sup        = <numpy.testing._private.utils.suppress_warnings object at 0x000001EE053C82B0>
sz         = 15

numpy\core\tests\test_umath.py:1319: AssertionError
_______________________ TestMinMax.test_reduce_reorder ________________________

self = <numpy.core.tests.test_umath.TestMinMax object at 0x000001EE01ECAE80>

    def test_reduce_reorder(self):
        # gh 10370, 11029 Some compilers reorder the call to npy_getfloatstatus
        # and put it before the call to an intrisic function that causes
        # invalid status to be set. Also make sure warnings are not emitted
        for n in (2, 4, 8, 16, 32):
            for dt in (np.float32, np.float16, np.complex64):
                for r in np.diagflat(np.array([np.nan] * n, dtype=dt)):
>                   assert_equal(np.min(r), np.nan)
E                   AssertionError:
E                   Items are not equal:
E                    ACTUAL: 0.0
E                    DESIRED: nan

dt         = <class 'numpy.float32'>
n          = 2
r          = array([ 0., nan], dtype=float32)
self       = <numpy.core.tests.test_umath.TestMinMax object at 0x000001EE01ECAE80>

numpy\core\tests\test_umath.py:1341: AssertionError
_________________ TestNanFunctions_MinMax.test_result_values __________________

self = <numpy.lib.tests.test_nanfunctions.TestNanFunctions_MinMax object at 0x000001EE0422BC88>

    def test_result_values(self):
        for nf, rf in zip(self.nanfuncs, self.stdfuncs):
            tgt = [rf(d) for d in _rdat]
>           res = nf(_ndat, axis=1)

nf         = <function nanmin at 0x000001EE78A32F28>
rf         = <function amin at 0x000001EE78531D90>
self       = <numpy.lib.tests.test_nanfunctions.TestNanFunctions_MinMax object at 0x000001EE0422BC88>
tgt        = [0.0116, -0.9403, -0.5954, 0.161]

numpy\lib\tests\test_nanfunctions.py:83:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

a = array([[ 0.6244,     nan,  0.2692,  0.0116,     nan,  0.117 ],
       [ 0.5351, -0.9403,     nan,  0.21  ,  0.4759,  0...[    nan,     nan,     nan,  0.1042,     nan, -0.5954],
       [ 0.161 ,     nan,     nan,  0.1859,  0.3146,     nan]])
axis = 1, out = None, keepdims = <no value>

    @array_function_dispatch(_nanmin_dispatcher)
    def nanmin(a, axis=None, out=None, keepdims=np._NoValue):
        """
        Return minimum of an array or minimum along an axis, ignoring any NaNs.
        When all-NaN slices are encountered a ``RuntimeWarning`` is raised and
        Nan is returned for that slice.

        Parameters
        ----------
        a : array_like
            Array containing numbers whose minimum is desired. If `a` is not an
            array, a conversion is attempted.
        axis : {int, tuple of int, None}, optional
            Axis or axes along which the minimum is computed. The default is to compute
            the minimum of the flattened array.
        out : ndarray, optional
            Alternate output array in which to place the result.  The default
            is ``None``; if provided, it must have the same shape as the
            expected output, but the type will be cast if necessary.  See
            `doc.ufuncs` for details.

            .. versionadded:: 1.8.0
        keepdims : bool, optional
            If this is set to True, the axes which are reduced are left
            in the result as dimensions with size one. With this option,
            the result will broadcast correctly against the original `a`.

            If the value is anything but the default, then
            `keepdims` will be passed through to the `min` method
            of sub-classes of `ndarray`.  If the sub-classes methods
            does not implement `keepdims` any exceptions will be raised.

            .. versionadded:: 1.8.0

        Returns
        -------
        nanmin : ndarray
            An array with the same shape as `a`, with the specified axis
            removed.  If `a` is a 0-d array, or if axis is None, an ndarray
            scalar is returned.  The same dtype as `a` is returned.

        See Also
        --------
        nanmax :
            The maximum value of an array along a given axis, ignoring any NaNs.
        amin :
            The minimum value of an array along a given axis, propagating any NaNs.
        fmin :
            Element-wise minimum of two arrays, ignoring any NaNs.
        minimum :
            Element-wise minimum of two arrays, propagating any NaNs.
        isnan :
            Shows which elements are Not a Number (NaN).
        isfinite:
            Shows which elements are neither NaN nor infinity.

        amax, fmax, maximum

        Notes
        -----
        NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic
        (IEEE 754). This means that Not a Number is not equivalent to infinity.
        Positive infinity is treated as a very large number and negative
        infinity is treated as a very small (i.e. negative) number.

        If the input has a integer type the function is equivalent to np.min.

        Examples
        --------
        >>> a = np.array([[1, 2], [3, np.nan]])
        >>> np.nanmin(a)
        1.0
        >>> np.nanmin(a, axis=0)
        array([ 1.,  2.])
        >>> np.nanmin(a, axis=1)
        array([ 1.,  3.])

        When positive infinity and negative infinity are present:

        >>> np.nanmin([1, 2, np.nan, np.inf])
        1.0
        >>> np.nanmin([1, 2, np.nan, np.NINF])
        -inf

        """
        kwargs = {}
        if keepdims is not np._NoValue:
            kwargs['keepdims'] = keepdims
        if type(a) is np.ndarray and a.dtype != np.object_:
            # Fast, but not safe for subclasses of ndarray, or object arrays,
            # which do not implement isnan (gh-9009), or fmin correctly (gh-8975)
            res = np.fmin.reduce(a, axis=axis, out=out, **kwargs)
            if np.isnan(res).any():
>               warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=2)
E               RuntimeWarning: All-NaN slice encountered

a          = array([[ 0.6244,     nan,  0.2692,  0.0116,     nan,  0.117 ],
       [ 0.5351, -0.9403,     nan,  0.21  ,  0.4759,  0...[    nan,     nan,     nan,  0.1042,     nan, -0.5954],
       [ 0.161 ,     nan,     nan,  0.1859,  0.3146,     nan]])
axis       = 1
keepdims   = <no value>
kwargs     = {}
out        = None
res        = array([ 0.0116, -0.9403,     nan,  0.161 ])

numpy\lib\nanfunctions.py:294: RuntimeWarning
_____________________ TestNanFunctions_MinMax.test_masked _____________________

self = <numpy.lib.tests.test_nanfunctions.TestNanFunctions_MinMax object at 0x000001EE04713240>

    def test_masked(self):
        mat = np.ma.fix_invalid(_ndat)
        msk = mat._mask.copy()
        for f in [np.nanmin]:
            res = f(mat, axis=1)
>           tgt = f(_ndat, axis=1)

f          = <function nanmin at 0x000001EE78A32F28>
mat        = masked_array(
  data=[[0.6244, --, 0.2692, 0.0116, --, 0.117],
        [0.5351, -0.9403, --, 0.21, 0.4759, 0.2833],
  ...   [ True,  True,  True, False,  True, False],
        [False,  True,  True, False, False,  True]],
  fill_value=1e+20)
msk        = array([[False,  True, False, False,  True, False],
       [False, False,  True, False, False, False],
       [ True,  True,  True, False,  True, False],
       [False,  True,  True, False, False,  True]])
res        = masked_array(data=[0.0116, -0.9403, -0.5954, 0.161],
             mask=[False, False, False, False],
       fill_value=1e+20)
self       = <numpy.lib.tests.test_nanfunctions.TestNanFunctions_MinMax object at 0x000001EE04713240>

numpy\lib\tests\test_nanfunctions.py:107:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

a = array([[ 0.6244,     nan,  0.2692,  0.0116,     nan,  0.117 ],
       [ 0.5351, -0.9403,     nan,  0.21  ,  0.4759,  0...[    nan,     nan,     nan,  0.1042,     nan, -0.5954],
       [ 0.161 ,     nan,     nan,  0.1859,  0.3146,     nan]])
axis = 1, out = None, keepdims = <no value>

    @array_function_dispatch(_nanmin_dispatcher)
    def nanmin(a, axis=None, out=None, keepdims=np._NoValue):
        """
        Return minimum of an array or minimum along an axis, ignoring any NaNs.
        When all-NaN slices are encountered a ``RuntimeWarning`` is raised and
        Nan is returned for that slice.

        Parameters
        ----------
        a : array_like
            Array containing numbers whose minimum is desired. If `a` is not an
            array, a conversion is attempted.
        axis : {int, tuple of int, None}, optional
            Axis or axes along which the minimum is computed. The default is to compute
            the minimum of the flattened array.
        out : ndarray, optional
            Alternate output array in which to place the result.  The default
            is ``None``; if provided, it must have the same shape as the
            expected output, but the type will be cast if necessary.  See
            `doc.ufuncs` for details.

            .. versionadded:: 1.8.0
        keepdims : bool, optional
            If this is set to True, the axes which are reduced are left
            in the result as dimensions with size one. With this option,
            the result will broadcast correctly against the original `a`.

            If the value is anything but the default, then
            `keepdims` will be passed through to the `min` method
            of sub-classes of `ndarray`.  If the sub-classes methods
            does not implement `keepdims` any exceptions will be raised.

            .. versionadded:: 1.8.0

        Returns
        -------
        nanmin : ndarray
            An array with the same shape as `a`, with the specified axis
            removed.  If `a` is a 0-d array, or if axis is None, an ndarray
            scalar is returned.  The same dtype as `a` is returned.

        See Also
        --------
        nanmax :
            The maximum value of an array along a given axis, ignoring any NaNs.
        amin :
            The minimum value of an array along a given axis, propagating any NaNs.
        fmin :
            Element-wise minimum of two arrays, ignoring any NaNs.
        minimum :
            Element-wise minimum of two arrays, propagating any NaNs.
        isnan :
            Shows which elements are Not a Number (NaN).
        isfinite:
            Shows which elements are neither NaN nor infinity.

        amax, fmax, maximum

        Notes
        -----
        NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic
        (IEEE 754). This means that Not a Number is not equivalent to infinity.
        Positive infinity is treated as a very large number and negative
        infinity is treated as a very small (i.e. negative) number.

        If the input has a integer type the function is equivalent to np.min.

        Examples
        --------
        >>> a = np.array([[1, 2], [3, np.nan]])
        >>> np.nanmin(a)
        1.0
        >>> np.nanmin(a, axis=0)
        array([ 1.,  2.])
        >>> np.nanmin(a, axis=1)
        array([ 1.,  3.])

        When positive infinity and negative infinity are present:

        >>> np.nanmin([1, 2, np.nan, np.inf])
        1.0
        >>> np.nanmin([1, 2, np.nan, np.NINF])
        -inf

        """
        kwargs = {}
        if keepdims is not np._NoValue:
            kwargs['keepdims'] = keepdims
        if type(a) is np.ndarray and a.dtype != np.object_:
            # Fast, but not safe for subclasses of ndarray, or object arrays,
            # which do not implement isnan (gh-9009), or fmin correctly (gh-8975)
            res = np.fmin.reduce(a, axis=axis, out=out, **kwargs)
            if np.isnan(res).any():
>               warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=2)
E               RuntimeWarning: All-NaN slice encountered

a          = array([[ 0.6244,     nan,  0.2692,  0.0116,     nan,  0.117 ],
       [ 0.5351, -0.9403,     nan,  0.21  ,  0.4759,  0...[    nan,     nan,     nan,  0.1042,     nan, -0.5954],
       [ 0.161 ,     nan,     nan,  0.1859,  0.3146,     nan]])
axis       = 1
keepdims   = <no value>
kwargs     = {}
out        = None
res        = array([ 0.0116, -0.9403,     nan,  0.161 ])

numpy\lib\nanfunctions.py:294: RuntimeWarning
8 failed, 6718 passed, 354 skipped, 167 deselected, 10 xfailed in 175.58 seconds

Minimal repro:

>>> np.maximum(0, np.nan)
0   # my machine
nan  # everywhere else

The text was updated successfully, but these errors were encountered:

eric-wieser · 2018-12-09T21:54:02Z

Reproduces after deleting the build directory and rebuilding too.

charris · 2018-12-09T21:58:03Z

Try undefining __AVX2__ in simd.inc.src.

charris · 2018-12-09T21:59:36Z

~~NVM, this is something else.~~ or maybe not :)

eric-wieser · 2018-12-09T22:01:04Z

This perhaps resembles #7601

eric-wieser · 2018-12-09T22:03:10Z

Try undefining __AVX2__ in simd.inc.src.

Doesn't help

charris · 2018-12-09T22:06:07Z

It's annoying that these problems are slipping through the CI testing.

eric-wieser · 2018-12-09T22:17:11Z

More updates:

de7036c (BUG: Do not double-quote arguments to the command line #12413) is broken too
d1e0a43 is also broken (the next oldest commit that I can build at all)
3430d78 (1.15.4) is fine - so this isn't MSVC silently changing under the hood on me - we've broken it as part of 1.16 somehow

charris · 2018-12-09T22:31:59Z

I assume master is also broken?

eric-wieser · 2018-12-09T22:32:59Z

The commit I built was master in the very recent past. I also assume it is, but haven't gone as far as checking.

charris · 2018-12-09T22:35:34Z

1.15.4 is pretty old, there have been ~400 PR's since. The micro versions aren't new branches off master.

eric-wieser · 2018-12-09T22:36:27Z

Right - but 1.16.x is the branch that I'm reporting at the top of this PR, which is very recent.

eric-wieser · 2018-12-09T23:08:16Z

Resolution: #12236 is to blame, specifically 24c9faa. cc @mattip

tylerjereddy · 2018-12-09T23:54:18Z

It's annoying that these problems are slipping through the CI testing.

Yes, suggestions for preventing this are most welcome of course. It is, however, also true that it is quite easy to "break" NumPy while having the CI all green.

Can we make the CI testing sensitive to this low-level issue in a straightforward way? What's happening on your machine that isn't happening in CI?

eric-wieser · 2018-12-10T00:01:37Z

#12411 failed on my machine but worked on azure because python and msvc are installed in a path with spaces on my machine..

Some possible causes of my floating point behavior differing from CI in this case:

using runtests.py to build instead of python setup.py is affecting the optimization flags
using MSVC 2015 - it looks like the CI job uses MSVC 2017 for all versions of python 3.

Perhaps we should add vs2015-win2012r2 to our matrix for python 3.5?

tylerjereddy · 2018-12-10T00:12:45Z

Thanks Eric, I've made a note of these ideas; I can test a few on my fork to see if I can get reliable failures.

charris · 2018-12-10T01:25:43Z

also true that it is quite easy to "break" NumPy while having the CI all green.

That's what the *.0 releases are for :) I figured 1.16 was going to be a difficult release, I just didn't expect the troubles to start quite this soon.

mattip · 2018-12-10T02:49:24Z

@eric-wieser can you attach the build.log? I cannot reproduce using MSVC\14.11.25503\bin\HostX64\x64\cl.exe

mattip · 2018-12-10T02:55:19Z

Where can you even download msvc 2015? There used to be a link in https://visualstudio.microsoft.com/downloads/ -> All Downloads -> Other Tools, but it seems to be gone now

tylerjereddy · 2018-12-10T03:42:07Z

Azure can pull in msvc 2015, but for Python 3.5 I see Version spec 3.5 for architecture x64 did not match any version in Agent.ToolsDirectory. I'm trying with 3.6 now, if that's of any use, otherwise we'd need to install Python 3.5 via another route.

The image code is open source I think ( https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/win/Vs2015-Server2012R2-Readme.md ) -- can perhaps grep around in that repo to find it.

mattip · 2018-12-10T04:25:33Z

@eric-wieser can you check if adding an exception for that compiler to the loop in loops.c.src:1861 helps? I am not sure what path through the loop the cases take, perhaps it would be enough to disable the simd part of the loop. Note we had to work around a similar bug in #7678, here is the code that was added there

/*
 * Visual Studio 2015 loop vectorizer handles NaN in an unexpected
 * manner, see: https://github.com/numpy/numpy/issues/7601
 */
#if (_MSC_VER == 1900)
#pragma loop( no_vector )
#endif

eric-wieser · 2018-12-10T07:12:52Z

@mattip, here's my build.log

I tried inserting that pragma (as well as playing with float_control, but it didn't seem to have any effect:

diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src
index f96e621b8..7a3824846 100644
--- a/numpy/core/src/umath/loops.c.src
+++ b/numpy/core/src/umath/loops.c.src
@@ -132,6 +132,20 @@
     npy_intp i;\
     for(i = 0; i < n; i++, ip1 += is1, ip2 += is2, op1 += os1)

+// https://github.com/numpy/numpy/issues/7601
+#if _MSC_VER == 1900
+    #define BINARY_LOOP_NO_VECTOR_MSVC_1900\
+        char *ip1 = args[0], *ip2 = args[1], *op1 = args[2];\
+        npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];\
+        npy_intp n = dimensions[0];\
+        npy_intp i;\
+        __pragma(loop(no_vector)) \
+        for(i = 0; i < n; i++, ip1 += is1, ip2 += is2, op1 += os1)
+#else
+    #define BINARY_LOOP_NO_VECTOR_MSVC_1900 BINARY_LOOP
+#endif
+
+
 /*
  * loop with contiguous specialization
  * op should be the code working on `tin in1`, `tin in2` and
@@ -1849,6 +1863,7 @@ NPY_NO_EXPORT void
     }
 }

+#pragma float_control(precise, on, push)
 /**begin repeat1
  * #kind = maximum, minimum#
  * #OP =  >=, <=#
@@ -1867,7 +1882,7 @@ NPY_NO_EXPORT void
         }
     }
     else {
-        BINARY_LOOP {
+        BINARY_LOOP_NO_VECTOR_MSVC_1900 {
             @type@ in1 = *(@type@ *)ip1;
             const @type@ in2 = *(@type@ *)ip2;
             in1 = (npy_isnan(in1) || in1 @OP@ in2) ? in1 : in2;
@@ -1894,7 +1909,7 @@ NPY_NO_EXPORT void
         *((@type@ *)iop1) = io1;
     }
     else {
-        BINARY_LOOP {
+        BINARY_LOOP_NO_VECTOR_MSVC_1900 {
             const @type@ in1 = *(@type@ *)ip1;
             const @type@ in2 = *(@type@ *)ip2;
             *((@type@ *)op1) = (npy_isnan(in2) || in1 @OP@ in2) ? in1 : in2;
@@ -1903,6 +1918,7 @@ NPY_NO_EXPORT void
     npy_clear_floatstatus_barrier((char*)dimensions);
 }
 /**end repeat1**/
+#pragma float_control(pop)

 NPY_NO_EXPORT void
 @TYPE@_floor_divide(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func))
@@ -2769,6 +2785,7 @@ NPY_NO_EXPORT void
     }
 }

+#pragma float_control(precise, on, push)
 /**begin repeat1
  * #kind = maximum, minimum#
  * #OP = CGE, CLE#
@@ -2776,7 +2793,7 @@ NPY_NO_EXPORT void
 NPY_NO_EXPORT void
 @TYPE@_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func))
 {
-    BINARY_LOOP {
+    BINARY_LOOP_NO_VECTOR_MSVC_1900 {
         @ftype@ in1r = ((@ftype@ *)ip1)[0];
         @ftype@ in1i = ((@ftype@ *)ip1)[1];
         const @ftype@ in2r = ((@ftype@ *)ip2)[0];
@@ -2799,7 +2816,7 @@ NPY_NO_EXPORT void
 NPY_NO_EXPORT void
 @TYPE@_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func))
 {
-    BINARY_LOOP {
+    BINARY_LOOP_NO_VECTOR_MSVC_1900 {
         const @ftype@ in1r = ((@ftype@ *)ip1)[0];
         const @ftype@ in1i = ((@ftype@ *)ip1)[1];
         const @ftype@ in2r = ((@ftype@ *)ip2)[0];
@@ -2816,6 +2833,7 @@ NPY_NO_EXPORT void
     npy_clear_floatstatus_barrier((char*)dimensions);
 }
 /**end repeat1**/
+#pragma float_control(pop)

 #define @TYPE@_true_divide @TYPE@_divide

mattip · 2018-12-13T20:06:31Z

Fixed via #12526

eric-wieser added the 00 - Bug label Dec 9, 2018

eric-wieser added this to the 1.16.0 release milestone Dec 9, 2018

eric-wieser added the 06 - Regression label Dec 9, 2018

charris added the 09 - Backport-Candidate PRs tagged should be backported label Dec 10, 2018

eric-wieser mentioned this issue Dec 10, 2018

[Windows Python 3.5 only] np.clip replace nans with lower bound #7601

Closed

eric-wieser changed the title ~~1.16 maintenance branch fails tests on windows~~ 1.16 maintenance branch fails tests on MSVC 2015 Dec 10, 2018

mattip mentioned this issue Dec 10, 2018

BUG: reorder operations for VS2015 #12526

Merged

tylerjereddy mentioned this issue Dec 10, 2018

TST: add MSVC 2015 / Python 3.5 to CI #12529

Closed

eric-wieser changed the title ~~1.16 maintenance branch fails tests on MSVC 2015~~ 1.16 maintenance branch fails tests on MSVC 2015 (VS2015 update 2) Dec 12, 2018

charris added the 26 - Compiler label Dec 13, 2018

mattip closed this as completed Dec 13, 2018

charris removed the 09 - Backport-Candidate PRs tagged should be backported label Dec 16, 2018

charris mentioned this issue Dec 16, 2018

BUG: reorder operations for VS2015 #12567

Merged

Uh oh!

1.16 maintenance branch fails tests on MSVC 2015 (VS2015 update 2) #12520

1.16 maintenance branch fails tests on MSVC 2015 (VS2015 update 2) #12520

Comments

eric-wieser commented Dec 9, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

eric-wieser commented Dec 9, 2018

Uh oh!

charris commented Dec 9, 2018

Uh oh!

charris commented Dec 9, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

eric-wieser commented Dec 9, 2018

Uh oh!

eric-wieser commented Dec 9, 2018

Uh oh!

charris commented Dec 9, 2018

Uh oh!

eric-wieser commented Dec 9, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

charris commented Dec 9, 2018

Uh oh!

eric-wieser commented Dec 9, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

charris commented Dec 9, 2018

Uh oh!

eric-wieser commented Dec 9, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

eric-wieser commented Dec 9, 2018

Uh oh!

tylerjereddy commented Dec 9, 2018

Uh oh!

eric-wieser commented Dec 10, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

tylerjereddy commented Dec 10, 2018

Uh oh!

charris commented Dec 10, 2018

Uh oh!

mattip commented Dec 10, 2018

Uh oh!

mattip commented Dec 10, 2018

Uh oh!

tylerjereddy commented Dec 10, 2018

Uh oh!

mattip commented Dec 10, 2018

Uh oh!

eric-wieser commented Dec 10, 2018

Uh oh!

mattip commented Dec 13, 2018

Uh oh!

eric-wieser commented Dec 9, 2018 •

edited

Loading

charris commented Dec 9, 2018 •

edited

Loading

eric-wieser commented Dec 9, 2018 •

edited

Loading

eric-wieser commented Dec 9, 2018 •

edited

Loading

eric-wieser commented Dec 9, 2018 •

edited

Loading

eric-wieser commented Dec 10, 2018 •

edited

Loading